#!/usr/bin/perl

#If $ARGV[0] equals 'CREATING' that means that you will be creating tables fresh, otherwise you will be updating existing tables.

use Cwd;

$baseurl = getcwd;


if ($ARGV[0] eq 'CREATING') {
    $mode = 'CREATING';
} else {
    $mode = 'UPDATING';
}

if ($ARGV[1] =~ /\//) {
    ($baseurl = $ARGV[1]) =~ s/[\/]*$//;
}

$error_outs = $baseurl . '/errors_Bibs.txt';
$print_outs = $baseurl . '/prints_Bibs.txt';

binmode(STDOUT, ':utf8');

open (STDERR, ">$error_outs");
open (STDOUT, ">$print_outs");

print "baseurl: " . $baseurl . "\n";

# local system variables
$connect = $baseurl . '/connect.pl';
require $connect;

use DBI;

use POSIX;

use IO::File;

use Data::Dumper;

use Roman; #for roman numerals

use utf8;

use Unicode::Normalize;

use MARC::Charset 'marc8_to_utf8';

use MARC::Batch;

#use strict;
#  use Carp ();
#    local $SIG{__WARN__} = \&Carp::cluck;

&hashes_and_arrays; #SUBROUTINE AT END OF SCRIPT

if ($ARGV[0] eq 'CREATING') {
$next_local_control_number = 1000001;
}

($HOUR, $MIN, $SEC) = (localtime)[2, 1, 0];
$start_time = "$HOUR:$MIN:$SEC";

$folder_name = 'bibs';

$where = $baseurl . '/' . $folder_name;

while (defined($next_file = <$where/*.out>)) {

    ($batch_textfiles = $next_file) =~ s/bibs/bibs_textfiles/g;

$batch_loader = IO::File->new("> $batch_textfiles") || die "cannot open $batch_textfiles for writing: $!\n";
binmode( $batch_loader, ':utf8' );
flock($batch_loader, 2);

    ($tracings_textfiles = $next_file) =~ s/bibs/tracings_textfiles/g;

$raw_tracings_loader = IO::File->new("> $tracings_textfiles") || die "cannot open $tracings_textfiles for writing: $!\n";
binmode( $raw_tracings_loader, ':utf8' );
flock($raw_tracings_loader, 2);


$edit1 = "perl -pi -e 's/\n|\t/\x1a/g' $next_file";
system($edit1);

print "$next_file\n";

    open(IN, "< $next_file") || die "cannot open $next_file for reading: $!";

            while(<IN>) {


$batch = MARC::Batch->new( 'USMARC', $next_file );
$batch->strict_off();
$batch->warnings_off();

while ($record = $batch->next()) {

@raw_tracings_array = ();

%query_insert = (
  source_record_type => 'MARC',
  source_record => '',
  local_control_number => '',
  record_owner => $institution,
  owner_level => '4',
  universal_identifier => '',
  national_control_number => '',  
  litform => '',
  roles => '',
  authors_personal => '',
  authors_corporate => '',
  authors_conferences => '',
  authors_uncontrolled => '',
  subjects_personal => '',
  subjects_corporate => '',
  subjects_conferences => '',
  subjects_titles => '',
  subjects_worktitles => '',
  subjects_topical => '',
  subjects_chronological => '',
  subjects_geographic => '',
  subjects_genre => '',
  topics_facet => '',
  geographics_facet => '',
  genres_facet => '',
  coverage_dates_facet => '',
  composition_dates_facet => '',
  instrumentations_facet => '',
  event_dates_facet => '',
  titles_uniform => '',
  titles_series => '',
  titles_analytic => '',
  titles_alternate => '',
  oclc => '',
  worktitles => '',
  mainheading => '',
  uniformtitle => '',
  maintitle => '',
  maintitle_article => '',
  title_proper => '',
  gmd => '',
  subtitle => '',
  statement_of_responsibility => '',
  languages => '',
  audience => 'Unspecified',
  pubplace_facet => '',
  tracings_titles_alternate => '',
  tracings_titles_analytic => '',
  tracings_titles_earlier => '',
  tracings_titles_later => '',
  tracings_added_authors => '',
  tracings_added_worktitles => '',
  edition => '',
  bib_status => '',
  cat_date => '',
  created_date => '', 
  updated_date => '', 
  isn => '',
  mattype => '',
  biblevel => '',
  composed_mattype => '',
  marc007_format => '',
  formats => '',
  accessmethods => '',
  datetype => '',
  original_pubdate_start => '',
  original_pubdate_end => '',
  original_pubdate => '',
  original_pubdate_range => '',
  original_pubdate_decades => '',
  original_pubdate_centuries => '',
  reprint_pubdate => '',
  imprint => '',
  pubplace => '',
  publisher => '',
  extent => '',
  roman_pages => '',
  pages => '',
  duration => '',
  series_traced => '',
  series_untraced => '',
  notes_public => '',
  notes_nonpublic => '',
  urls => '',
  lc_callnumber => '',
  lc_class => '',
  lcc_range => '',
  lc_class_label => '',
  sudoc_callnumber => '',
  sudoc_class => '',
  dewey_callnumber => '',
  dewey_class => '',
  other_callnumber => '',
  other_class => '',
  local_callnumber => '',
  local_class => '',
  call_numbers => '',
  bloc => '',
  donors => '',
  collections => '',
  unaccounted_marc_fields => '',
  marc_errors => ''
);

    $this_local_control_number = substr($record->field('907')->subfield('a'), 2, 7);

if ($ARGV[0] eq 'CREATING') {
##THIS STARTS WHERE THE DELETED BIB RECORDS ARE LOADED

#print "$next_local_control_number < $this_local_control_number\n";
    while($next_local_control_number < $this_local_control_number) {

        $query_insert{'local_control_number'} = $institution . 'b' . $next_local_control_number;
        $query_insert{'bib_status'} = 'x';

        @keys = keys %query_insert;
        @values = values %query_insert;

        $mysql_keys = join(", ", @keys);
        $mysql_values = join("\t", @values);
        $mysql_values =~ s/\\\t/\\ \t/g;

        print $batch_loader $mysql_values . "\n";

        $next_local_control_number++;

    }

    $next_local_control_number++;

##THIS ENDS WHERE THE DELETED BIB RECORDS ARE LOADED
}

$field_780_indicator2_4 = 0;
$field_785_indicator2_6 = 0;
$field_785_indicator2_7 = 0;

$this_local_control_number = $institution . 'b' . $this_local_control_number;
$query_insert{'universal_identifier'} = $this_local_control_number . "_";

$query_insert{'source_record'} = $record->as_usmarc();

if($query_insert{'source_record'} =~ /\x1a/) {
    print $this_local_control_number . " has an escape character (either a tab or line ending) in a field marked with \x1a \n";
    uniqueness('marc_errors', '', $this_local_control_number . " has an escape character (either a tab or line ending) in a field marked with \x1a");
}

if ($consortium ne '') {
    if(grep /$institution/, @MeSH_major_players) {
        $query_insert{'owner_level'} = '0';
    } elsif(grep /$institution/, @major_players) {
        $query_insert{'owner_level'} = '1';
    } elsif(grep /$institution/, @consortium_members) {
        $query_insert{'owner_level'} = '2';
    }
} else {
    $query_insert{'owner_level'} = '1';
}


foreach $fLDR ($record->leader()) {
    $query_insert{'mattype'} = $mattypeshash{substr($fLDR, 6, 1)};
    $query_insert{'biblevel'} = $biblevelshash{substr($fLDR, 7, 1)};
    $thismattype_code = substr($fLDR, 6, 2);

    if(exists($composed_mattypeshash{$thismattype_code})) {
        $query_insert{'composed_mattype'} = $composed_mattypeshash{substr($fLDR, 6, 2)};
    } else {
        $query_insert{'composed_mattype'} = $query_insert{'mattype'} . ' -- ' . $query_insert{'biblevel'};

    }

}

foreach $f001 ($record->field('001|035')) {
    $query_insert{'oclc'} = $f001->as_string;
    if($query_insert{'oclc'} =~ /^oc[mn]/) {
        $query_insert{'universal_identifier'} = $query_insert{'oclc'};
        $query_insert{'oclc'} =~ s/^oc[mn][0]*//;
    } else {
        $query_insert{'universal_identifier'} = $this_local_control_number . '_' . $query_insert{'oclc'};
    }
}

foreach $f245 ($record->field('245')) {
    foreach $f245h ($f245->subfield('h')) {
        ($this_gmd = $f245h) =~ s/[^ a-zA-Z0-9]//g;
        $this_gmd =~ s/ *$//g;
        $query_insert{"gmd"} = $this_gmd;
    }
}

foreach $f006 ($record->field('006')) {
    if($f006->as_string =~ /^m[ ]{8}d/) {
            uniqueness('formats', '', 'Electronic Resource');
    } else {
        # uniqueness('unaccounted_marc_fields', 'The 006 field does not match the formatting for \'Electronic Resource\' materials. Either the content (' . $record->field('006')->as_string . ') will need to be changed, or the parsing script will need to be modified');
    }
}

foreach $f007 ($record->field('007')) {
    $f007 = $f007->as_string;
    $positions = substr($f007, 0, 7);
    @position = split(//, $positions);

    if(exists($smdshash{$position[0]})) {
        uniqueness('formats', '', $smdshash{$position[0]});
    }

    uniqueness('marc007_format', '', $f007 . ' (' . $smdshash{$position[0]} . ')');

    if($position[0] =~ /r/) {
        if(exists($twofieldshash{$position[0]. $position[3]})) {
            uniqueness('formats', '', $twofieldshash{$position[0] . $position[3]});
        }
    } elsif($position[0] =~ /v/) {
        if(exists($twofieldshash{$position[0]. $position[4]})) {
            uniqueness('formats', '', $twofieldshash{$position[0] . $position[4]});
        }
    } elsif($position[0] =~ /s/) {
        if($position[1] =~ /d/) {
            if(exists($twofieldshash{$position[0]. $position[1] . $position[6]})) {
                uniqueness('formats', '', $twofieldshash{$position[0]. $position[1] . $position[6]});
            } else {
                uniqueness('formats', '', 'unknown disc');
            }
        } elsif($position[1] =~ /egiqstuwz/) {
            if(exists($twofieldshash{$position[0]. $position[1]})) {
                uniqueness('formats', '', $twofieldshash{$position[0]. $position[1]});
            }
        }
    } elsif($position[0] =~ /[acfghkm]/) {
        if(exists($twofieldshash{$position[0]. $position[1]})) {
            uniqueness('formats', '', $twofieldshash{$position[0]. $position[1]});
        }
    }

    if($position[0] =~ /c/) {
        if(exists($accessmethodshash{$position[1]})) {
            uniqueness('accessmethods', '', $accessmethodshash{$position[1]});
        }
    }

    if($record->field('245')) {
        if(($record->field('245')->subfield('h') =~ /electronic resource/) && ($query_insert{'formats'} =~ /Video Recording/) && ($query_insert{'formats'} =~ /Remote/)) { 
            uniqueness('formats', '', 'Streaming Video');
        }
    }

    if(($query_insert{'gmd'} =~ /videorecording/) && ($query_insert{'formats'} !~ /Video Recording/)) {
        uniqueness('formats', '', 'Video Recording');
    }

}

foreach $f008 ($record->field('008')->as_string) {

    if(substr($record->leader, 6, 2) =~ /[at][acdm]/) {
        $LitF = substr($f008, 33, 1);
        if($LitF =~ /[0eis]/) {
            uniqueness('litform', '', 'Non-fiction');
            uniqueness('genres_facet',  'GENERIC\\t', 'Non-fiction');
        } elsif($LitF =~ /[1c-fhjmpu]/) {
            uniqueness('litform', '', 'Fiction');
            uniqueness('genres_facet', 'GENERIC\\t', 'Fiction');
        }

        if($LitF =~ /[c-fh-jmpsu]/) {   
            uniqueness('litform', '', $litformshash{$LitF});
            uniqueness('genres_facet', 'GENERIC\\t', $litformshash{$LitF});
        } 
    }

    if(substr($record->leader, 6, 1) =~ /g/) {
        $Time = substr($f008, 18, 3);
        if($Time =~ /[0-9]{3}/) {
            $Time *= 60; 
            $query_insert{'duration'} = $Time;
        }
    }

    if(substr($record->leader, 6, 1) !~ /[ef]/) {
        if(substr($record->leader, 7, 1) !~ /[s]/) {
            $Audn = substr($f008, 22, 1);

            if($Audn =~ /a/) {
                $query_insert{'audience'} = 'Juvenile\\nPreschool (0-5 years old)';
            } elsif($Audn =~ /b/) {
                $query_insert{'audience'} = 'Juvenile\\nPrimary (6-8 years old)';
            } elsif($Audn =~ /c/) {
                $query_insert{'audience'} = 'Juvenile\\nPre-adolescent (9-13 years old)';
            } elsif($Audn =~ /d/) {
                $query_insert{'audience'} = 'Juvenile\\nAdolescent (14-17 years old)';
            } elsif($Audn =~ /e/) {
                $query_insert{'audience'} = 'Adult';
            } elsif($Audn =~ /f/) {
                $query_insert{'audience'} = 'Specialized';
            } elsif($Audn =~ /g/) {
                $query_insert{'audience'} = 'General';
            } elsif($Audn =~ /j/) {
                $query_insert{'audience'} = 'Juvenile';
            } elsif($Audn =~ / /) {
                $query_insert{'audience'} = 'Unspecified';
            }
        }
    }

    if(substr($f008, 17, 1) =~ /u/) {
        $query_insert{'pubplace_facet'} = 'United States';
    }

    $pubplace_facet_code = substr($f008, 15, 3);
    $pubplace_facet_code =~ s/[ ]//g;
    uniqueness('pubplace_facet', '', $countries_hash{$pubplace_facet_code});

    if($query_insert{'mattype'} =~ /(?:Printed music|Manuscript music|Musical sound recording)/) {
        if(substr($f008, 18, 2) !~ /uu|nn|mu/) {
            $this_composition = $compositions_hash{substr($f008, 18, 2)};
            if($this_composition ne '') {
                $query_insert{'genres_facet'} = 'COMPOSITION\\t' . $this_composition . '.';
            }
        }
    }

    if($languages_hash{substr($f008, 35, 3)} ne '') {
        $this_language = $languages_hash{substr($f008, 35, 3)};
        $query_insert{'languages'} = '\\t' . $this_language;
    }

###START: SETTING THE DATES
    $f008_datetype = substr($f008, 6, 1);
    $query_insert{'datetype'} = $f008_datetype;

    ($YEAR) = (localtime)[5];
    $nextyear = $YEAR + 1901;

    if($f008_datetype =~ /[ikmnq]/) {
        ($date1 = substr($f008, 7, 4)) =~ s/ |u|\|/0/g;
        ($date2 = substr($f008, 11, 4)) =~ s/ |u|\|/9/g;
        $date2 =~ s/9999/$nextyear/;
        $date1 =~ s/0000/$date2/;
    } elsif($f008_datetype =~ /[cdu]/) {
        ($date1 = substr($f008, 7, 4)) =~ s/ |u|\|/0/g;
        ($date2 = substr($f008, 11, 4)) =~ s/ |u|\|/9/g;
        $date2 =~ s/9999/$nextyear/;
        $date1 =~ s/0000/$date2/;
    } elsif($f008_datetype =~ /[e]/) {
        if(substr($f008, 11, 2) <= 12) {
            ($date1 = substr($f008, 7, 4) . '.' . substr($f008, 11, 2) . '.' . substr($f008, 13, 2)) =~ s/ |u|\|/0/g;
            ($date2 = substr($f008, 7, 4) . '.' . substr($f008, 11, 2) . '.' . substr($f008, 13, 2)) =~ s/ |u|\|/0/g;
        } else {
            ($date1 = substr($f008, 7, 4)) =~ s/ |u|\|/0/g;
            ($date2 = substr($f008, 11, 4)) =~ s/ |u|\|/9/g;
            if($date1 > $date2) {
                uniqueness('marc_errors', '', $this_local_control_number . " has a bad code in the DtSt (datetype): " . $f008_datetype . " should be an \'r\'");
            }
        }
    } elsif($f008_datetype =~ /[s]/) {
        ($date1 = substr($f008, 7, 4)) =~ s/ |u|\|/0/g;
        if(substr($f008, 11, 4) ne '    ') {
            uniqueness('marc_errors', '', $this_local_control_number . " has a bad code in the DtSt (datetype): " . $f008_datetype . " OR bad data in the second date field: " . substr($f008, 11, 4));
        }
        ($date2 = substr($f008, 11, 4)) =~ s/ |u|\|/9/g;
        $date1 =~ s/0000/$date2/;
        $date2 =~ s/9999/$date1/;
    } elsif($f008_datetype =~ /[prt]/) {
        ($date1 = substr($f008, 7, 4)) =~ s/ |u|\|/0/g;
        ($date2 = substr($f008, 11, 4)) =~ s/ |u|\|/0/g;
        $date1 =~ s/0000//;
        $date2 =~ s/0000/$date1/;
    } elsif($f008_datetype =~ /[b]/) {
        if(substr($f008, 7, 8) ne '        ') {
            ($date1 = substr($f008, 7, 4)) =~ s/ |u|\|/0/g;
            ($date2 = substr($f008, 11, 4)) =~ s/ |u|\|/9/g;
            $date2 =~ s/9999/$date1/;
        } else {
            $date1 = $record->field('046')->subfield('b') . 'B.C'; 
            $date2 = $record->field('046')->subfield('d') . 'B.C';
        }
    } else {
        ($date1 = substr($f008, 7, 4)) =~ s/ |u|\|/0/g;
        ($date2 = substr($f008, 11, 4)) =~ s/ |u|\|/9/g;
        $date2 =~ s/9999/$date1/;
        uniqueness('marc_errors', '', $this_local_control_number . " has a bad code in the DtSt (datetype): " . $f008_datetype);

    } 

    if($date1 =~ /[a-zA-Z]/) {
        $date1 = '';
    }

    if($date2 =~ /[a-zA-Z]/) {
        $date2 = '';
    }

    if($f008_datetype =~ /[prt]/) {
        $query_insert{'original_pubdate_start'} = $date2;
        $query_insert{'original_pubdate_end'} = $date2;

        $query_insert{'reprint_pubdate'} = $date1;
        $query_insert{'original_pubdate'} = $date2;
        uniqueness('original_pubdate_decades', '', substr($date2, 0, 3) . '0' . '-' . substr($date2, 0, 3) . '9');
        uniqueness('original_pubdate_centuries', '', substr($date2, 0, 2) . '00' . '-' . substr($date2, 0, 2) . '99');
    } else {
        $query_insert{'original_pubdate_start'} = $date1;
        $query_insert{'original_pubdate_end'} = $date2;
        $query_insert{'original_pubdate'} = $date1;     
 
        if($date1 ne $date2) {
            $query_insert{'original_pubdate_range'} = $date1 . '-' . $date2;
        }

        $query_insert{'original_pubdate_decades'} = substr($date1, 0, 3) . '0' . '-' . substr($date2, 0, 3) . '9';
        $query_insert{'original_pubdate_centuries'} = substr($date1, 0, 2) . '00' . '-' . substr($date2, 0, 2) . '99';
    }

###END: SETTING THE DATES

}

foreach $f01X ($record->field('01[056]|030')) {
    $indicator1 = $f01X->indicator(1);
    $indicator2 = $f01X->indicator(2);
    $this_tag = $f01X->tag;
    $this_label = '';

    if($this_tag =~ /010/) {
        $hash_tag = 'national_control_number';
        $codes = '[abz]';
        $this_label = 'LCCN: ';
    } elsif($this_tag =~ /015/) {
        $hash_tag = 'national_control_number';
        $codes = '[a]';
        if($indicator2 =~ /[ ]/) {
            $this_label = $f01X->subfield('2') . ': ';
        }
    } elsif($this_tag =~ /016/) {
        $hash_tag = 'national_control_number';
        $codes = '[az]'; 
        if($indicator2 =~ /7/) {
            $this_label = 'NLC: ';
        } elsif($indicator2 =~ /[ ]/) {
            $this_label = $f01X->subfield('2') . ': ';
        }
    } elsif($this_tag =~ /030/) {
        $hash_tag = 'national_control_number';
        $codes = '[a]';
        $this_label = 'CODEN: ';
    }

    foreach $f01X_subfields ($f01X->subfields()) {
        ($f01Xcode, $f01Xdata ) = @$f01X_subfields;
        if($f01Xcode =~ /[b]/) {
            $this_label = 'NUCMC: ';
        } elsif($f01Xcode =~ /[z]/) {
            $this_label = 'Invalid ' . $this_label;
        }

    uniqueness('national_control_number', $this_label . '\\t', $f01Xdata);
    uniqueness_diacritics('notes_public', "$this_label ", $f01Xdata);

}

}

foreach $f02X ($record->field('02[024578]')) {
    $indicator1 = $f02X->indicator(1);
    $indicator2 = $f02X->indicator(2);
    $this_tag = $f02X->tag;
    $this_label = '';
    $this_isn = '';
    $this_note = '';

    if($this_tag =~ /020/) {
        $hash_tag = 'isn';
        $codes = '[acz]';
        $this_label = 'ISBN: ';
    } elsif($this_tag =~ /022/) {
        $hash_tag = 'isn';
        $codes = '[ayz2]';
        $this_label = 'ISSN: ';
    } elsif($this_tag =~ /024/) {
        $hash_tag = 'isn';
        $codes = '[acdz2]';
    } elsif($this_tag =~ /025/) {
        $hash_tag = 'isn';
        $codes = '[a]';
    } elsif($this_tag =~ /027/) {
        $hash_tag = 'isn';
        $codes = '[az]';
    } elsif($this_tag =~ /028/) {
        $hash_tag = 'isn';
        $codes = '[ab]';
    }

    if($this_tag =~ /024/) {
        if($indicator1 =~ /0/) {
            $this_label = 'ISRC: '; #International Standard Recording Code (ISRC)
        } elsif($indicator1 =~ /1/) {
            $this_label = 'UPC: '; # Universal Product Code (UPC)
        } elsif($indicator1 =~ /2/) {
            $this_label = 'ISMN'; #International Standard Music Number (ISMN)
        } elsif($indicator1 =~ /3/) {
            $this_label = 'EAN: '; #International Article Number (EAN)
        } elsif($indicator1 =~ /4/) {
            $this_label = 'SICI: '; #Serial Item and Contribution Identifier (SICI)
        } elsif($indicator1 =~ /7/) {
            $this_label = $f02X->subfield('2') . ': '; #Source specified in subfield 2
        } elsif($indicator1 =~ /8/) {
            $this_label = ''; #Unspecified type of standard number or code
        }
    } elsif($this_tag =~ /025/) {
        $this_label = 'Overseas acquisition number: ';
    } elsif($this_tag =~ /027/) {
        $this_label = 'STRN: '; #Standard Technical Report Number
    } elsif($this_tag =~ /028/) {
        if($indicator1 =~ /0/) {
            $this_label = 'Issue number: ';
        } elsif($indicator1 =~ /1/) {
            $this_label = 'Matrix number: ';
        } elsif($indicator1 =~ /2/) {
            $this_label = 'Plate number: ';
        } elsif($indicator1 =~ /3/) {
            $this_label = 'Other music number: ';
        } elsif($indicator1 =~ /4/) {
            $this_label = 'Videorecording number: ';
        } elsif($indicator1 =~ /5/) {
            $this_label = 'Other publisher number: ';
        }
    }

    foreach $f02X_subfields ($f02X->subfields()) {
    ($f02Xcode, $f02Xdata ) = @$f02X_subfields;
            if($f02Xcode =~ /[ayz]/) {
                if($f02Xdata=~ /([0-9xX]{13}|[0-9xX]{10}|[0-9xX]{4}-[0-9xX]{4}).*/) {
                    $this_isn .= $1;
                } else {
                    $this_isn .= $f02Xdata;
                }
           
                if($f02Xcode =~ /[a]/) {
                    uniqueness('isn', $this_label . '\\t', $this_isn);

                    $this_note .= $f02Xdata;

                } elsif($f02Xcode =~ /[yz]/) {
                    uniqueness_diacritics('isn', 'Invalid ISN: ', $this_isn);
                    uniqueness_diacritics('notes_public', 'Invalid ISN: ', $f02Xdata);
                }
            } else {
                $this_note .= $f02Xdata;
            }
    }

    uniqueness_diacritics('notes_public', "$this_label ", $this_note);

}

foreach $f033 ($record->field('033')) {
    $indicator1 = $f033->indicator(1);
    $indicator2 = $f033->indicator(2);
    $hash_tag = 'event_dates_facet';

    foreach $f033_subfields ($f033->subfields()) {
        ($f033code, $f033data ) = @$f033_subfields;
        if($f033code =~ /a/) {
            if(length($f033data) > 6) {
                $this_day = substr($f033data, 6, 2);
                $this_day =~ s/^0//;
                $this_day = ' ' . $this_day;
            }
            if(length($f033data) > 4) {
                $this_month = ' (' . $months[substr($f033data, 4, 2)] . $this_day . ')';
            }
            if($indicator1 =~ /2/) {
                if($this_date_range ne '') {
                    $this_year_range .= '-' . substr($f033data, 0, 4);
                    $this_date_range .= '-' . substr($f033data, 0, 4) . $this_month;
                } else {
                    $this_year_range .= substr($f033data, 0, 4);
                    $this_date_range .= substr($f033data, 0, 4) . $this_month;
                }
            } else {
                uniqueness($hash_tag, '', substr($f033data, 0, 4) . $this_month);
                dates_expansion($hash_tag, 'GENERIC', substr($f033data, 0, 4));
            }
        } elsif($f033code =~ /b/) {
            foreach $i (0, 2) {
                $event_place_code = 'G' . $f033data;
                $this_event_place = $event_places_hash{$event_place_code};
                if($this_event_place ne '') {
                    uniqueness('geographics_facet', 'GENERIC\\t', $this_event_place);
                }
            }
        } elsif($f033code =~ /c/) {
            foreach $i (0, 2) {
                $event_subplace_code = $event_place_code . '.' . $f033data;
                $this_event_place = $event_places_hash{$event_subplace_code};
                if($this_event_place ne '') {
                    uniqueness('geographics_facet', 'GENERIC\\t', $this_event_place);
                }
            }
        }

    }
        if($indicator1 =~ /2/) {
            dates_expansion($hash_tag, 'GENERIC', $this_year_range);
            uniqueness($hash_tag, 'GENERIC\\t', $this_date_range);
            undef($this_year_range);
            undef($this_date_range);
        }

}


foreach $f041 ($record->field('041')) {
    $indicator1 = $f041->indicator(1);
    $field_041_indicator1 = 0;
    my $language = '';

    foreach $f041_subfields ($f041->subfields()) {
        ($f041code, $f041data ) = @$f041_subfields;
        while($f041data =~ /([a-z]{3})/g) {
            $this_label = '\\t';
            $three_letters = $1;
            $language = $languages_hash{$three_letters};

            if($language ne '') {

                if($f041code =~ /[a]/) {
                    $this_label = '\\t';
                    if(($field_041_indicator1 > 0) && ($indicator1 == 1)) {
                        $this_label = ' (translated from)';
                    }
                    $field_041_indicator1++;                    
                } elsif($f041code =~ /[b]/) {
                    if(substr($record->leader, 7, 1) =~ /[g]/) {
                        $this_label = 'Subtitles are in \\t';
                    } else {
                        $this_label = 'Summary is in \\t';
                    }
                } elsif($f041code =~ /[d]/) {
                    $this_label = 'The recording is in \\t';
                } elsif($f041code =~ /[e]/) {
                    $this_label = 'Librettos in \\t';
                } elsif($f041code =~ /[f]/) {
                    $this_label = 'Table of contents in \\t';
                } elsif($f041code =~ /[g]/) {
                    $this_label = 'Accompanying material in \\t';
                } elsif($f041code =~ /[h]/) {
                    $this_label = ' (translated from)';
                }
                
                if($this_label eq ' (translated from)') {
                    uniqueness('languages', '\\t', $language . $this_label);
                } else {
                    uniqueness('languages', $this_label, $language);
                }
            }

        }
    }
}

foreach $f043 ($record->field('043')) {
    foreach $f043_subfields ($f043->subfields()) {
        ($f043code, $f043data ) = @$f043_subfields;
        if($f043code =~ /a/) {
            $f043data =~ s/[-]*$//;
            $this_geographic = $geographics_hash{$f043data};
            if($this_geographic ne '') {
                uniqueness('geographics_facet', 'LCSH\\t', $this_geographic);
            }
        }
    }
}

foreach $f044 ($record->field('044')) {
    foreach $f044_subfields ($f044->subfields()) {
        ($f044code, $f044data ) = @$f044_subfields;
        if($f044code =~ /a/) {
            if(substr($f044data, 2, 1) =~ /u/) {
                uniqueness('pubplace_facet', '', 'United States');
            }

            $pubplace_facet_code = $f044data;
            $pubplace_facet_code =~ s/[\s]*//;
            $query_insert{'pubplace_facet'} = $countries_hash{$pubplace_facet_code};

        }
    }
}

foreach $f045 ($record->field('045')) {
    $indicator1 = $f045->indicator(1);
    $indicator2 = $f045->indicator(2);
    $hash_tag = 'coverage_dates_facet';
    @these_coverage_dates = ();
    @these_date_ranges = ();
    @these_eras = ();


    if($query_insert{'mattype'} =~ /(?:Printed music|Manuscript music|Musical sound recording)/) {
        $hash_tag = 'composition_dates_facet';
    }

    foreach $f045_subfields ($f045->subfields()) {
        ($f045code, $f045data ) = @$f045_subfields;
        if($f045code =~ /a/) {
                if(substr($f045data, 0, 1) =~ /[a-d]/) {
                    $era1 = ' B.C';
                } else {
                    $era1 = ' A.D';
                }
                if(substr($f045data, 2, 1) =~ /[a-d]/) {
                    $era2 = ' B.C';
                } else {
                    $era2 = ' A.D';
                }

                if(($era1 eq ' A.D') && ($era2 eq ' A.D')) {
                    $era1 = '';
                    $era2 = '';
                }

            $f045data =~ tr/A-Z/a-z/;
            if($f045data =~ /[a-z][\-0-9][a-z][\-0-9]/) {
            dates_expansion($hash_tag, 'GENERIC', substr($coverage_dates_hash{substr($f045data, 0, 2)}, 0, 4) . $era1 . '-' . substr($coverage_dates_hash{substr($f045data, 2, 2)}, 5, 4) . $era2);
            } else {
                uniqueness('marc_errors', '', $this_local_control_number . " has a bad code in the 045: " . $f045data);
            }
        } elsif($f045code =~ /[bc]/) {

            if($f045data =~ /([cd]{0,1})([\D])*([0-9]*)/) {
                $this_epoch = $1;
                $this_coverage_date = $3;
            }

            if($this_epoch =~ /c/) {
                $era = ' B.C';
            } elsif($this_epoch =~ /d/) {
                if($indicator1 =~ /2/) {
                    $era = ' A.D';
                } else {
                    $era = '';
                }
            }

            if($indicator1 =~ /2/) {
                push(@these_date_ranges, substr($this_coverage_date, 0, 4));
                push(@these_eras, $era);
                push(@these_coverage_dates, $this_coverage_date);
            } else {
                uniqueness($hash_tag, 'GENERIC\\t', $this_coverage_date . $era);
                dates_expansion($hash_tag, 'GENERIC', substr($this_coverage_date, 0, 4) . $era);
            }
        }
    }

    if(($indicator1 =~ /2/) && ($#these_eras > 0)) {
                if(($these_eras[0] eq ' A.D') && ($these_eras[1] eq ' A.D')) {
                    $these_eras[0] = '';
                    $these_eras[1] = '';
                }

        uniqueness($hash_tag, 'GENERIC\\t', $these_coverage_dates[0] . $these_eras[0] . '-' . $these_coverage_dates[1] . $these_eras[1]);

        dates_expansion($hash_tag, 'GENERIC', $these_date_ranges[0] . $these_eras[0] . '-' . $these_date_ranges[1] . $these_eras[1]);
    }
}


foreach $f047 ($record->field('047')) {
    $indicator1 = $f047->indicator(1);
    $indicator2 = $f047->indicator(2);
    $this_tag = $f047->tag();

    foreach $f047_subfields ($f047->subfields()) {
        ($f047code, $f047data ) = @$f047_subfields;
        if($f047code =~ /a/) {
            $this_composition = $compositions_hash{$f047data};
            if($this_composition eq '') {
                uniqueness('genres_facet', 'COMPOSITION\\t', $f047data . '.');
            } else {
                uniqueness('genres_facet', 'COMPOSITION\\t', $this_composition . '.');
            }
        } 
    }
}

foreach $f048 ($record->field('048')) {
    $indicator1 = $f048->indicator(1);
    $indicator2 = $f048->indicator(2);
    $this_tag = $f048->tag();
    $this_parts = '';

    foreach $f048_subfields ($f048->subfields()) {
        ($f048code, $f048data ) = @$f048_subfields;
        if($f048code =~ /[ab]/) {
            $this_instrumentation = $instrumentations_hash{lc(substr($f048data, 0, 2))};

                if(length($f048data) == 4) {
                    $this_parts = substr($f048data, 2, 2);
                    $this_parts =~ s/^0//;
                    $this_parts = ' (' . $this_parts . ')';
                }

            if($this_instrumentation eq '') {
                uniqueness('instrumentations_facet', 'FORMATTED\\t', $f048data . $this_parts);
            } else {
                uniqueness('instrumentations_facet', 'FORMATTED\\t', $this_instrumentation . $this_parts);
                if($f048code =~ /[b]/) {
                    uniqueness('instrumentations_facet', 'FORMATTED\\t', $this_instrumentation . ', Accompanied soloist' . $this_parts);
                }
            }
        } elsif($f048code =~ /[2]/) {
            #SOURCE OF CODE
        }
    }
}


foreach $f05X ($record->field('050|060|070|086|09[0269]')) {
    $indicator1 = $f05X->indicator(1);
    $indicator2 = $f05X->indicator(2);
    $this_tag = $f05X->tag;
    $this_hash = '';

    if($this_tag =~ /050|090/) {
        $hash_tag = 'lc_callnumber';
        $class = 'lc_class';
        $codes = '[ab]';
        $label = 'LCC';
    } elsif($this_tag =~ /060|096/) { #NLM
        $hash_tag = 'other_callnumber';
        $class = 'other_class';
        $codes = '[ab]';
        $label = 'NLM';
    } elsif($this_tag =~ /070/) { #NAL
        $hash_tag = 'other_callnumber';
        $class = 'other_class';
        $codes = '[ab]';
        $label = 'NAL';
    } elsif($this_tag =~ /082|092/) {
        $hash_tag = 'dewey_callnumber';
        $class = 'dewey_class';
        $codes = '[ab]';
        $label = 'DEWEY';
    } elsif($this_tag =~ /086/) {
        $hash_tag = 'sudoc_callnumber';
        $class = 'sudoc_class';
        $codes = '[ab]';
        $label = 'SUDOC';
    } elsif($this_tag =~ /099/) {
        $hash_tag = 'local_callnumber';
        $class = 'local_class';
        $codes = '[ab]';
        $label = 'LOCAL';
    }

    if($this_tag !~ /050|090/) {
    uniqueness_diacritics($hash_tag, '', $f05X->as_string);
    uniqueness_diacritics('call_numbers', $label, $f05X->as_string);
    }
    
    if($f05X->as_string =~ /^[a-zA-Z]/) {
        ($this_class) = ($f05X->as_string =~ /(^[a-zA-Z]*)/);
        for($c = 1; $c <= length($this_class); $c++) {
            uniqueness_diacritics($class, '', substr($this_class, 0, $c));
        }
    } else {
        ($this_class) = ($f05X->as_string =~ /(^[0-9]{1})/);
        uniqueness_diacritics($class, '', $this_class . '00');
    }

    if($this_tag =~ /050|090/) {

                $decimal_point = '';
                $decimal_alpha = '';
                $decimal_point2 = '';

        if($f05X->subfield('a') =~ /(^[a-zA-Z]*)([0-9]*)([0-9a-zA-Z. ]*)/) {
            $this_class = $1 . (' ' x (3 - length($1)));
            $lc_numeric = (' ' x (4 - length($2))) . $2;
            $decimal_points = $3;

            if($decimal_points =~ /(\.)*([0-9]*)(\.)*([a-zA-Z0-9]*)/) {
                $decimal_point = $2;
                $decimal_point0 = $4;
                if($decimal_point0 =~ /([a-zA-Z]*)([0-9]*)/) {
                    $decimal_alpha = $1;
                    $decimal_point2 = $2;
                }
            } else {
                $decimal_point = substr($decimal_points, 0, 5);
                $decimal_alpha = '';
                $decimal_point2 = '';
            }

            $decimal_point .= ' ' x (5 - length($decimal_point));
            $decimal_alpha .= ' ' x (1 - length($decimal_alpha));
            $decimal_point2 .= ' ' x (6 - length($decimal_point2));

        }

        if($f05X->subfield('b')) {
            $cutter = $f05X->subfield('b');
        }

        $lc_callnumber = $this_class . $lc_numeric . "." . $decimal_point . "." . $decimal_alpha . $decimal_point2 . $cutter;
        $lc_callnumber =~ s/\. / /g;
        uniqueness_diacritics($hash_tag, '', $lc_callnumber);
        uniqueness_diacritics('call_numbers', $label, $lc_callnumber);

        $lcc_class = $this_class . $lc_numeric . $decimal_point . $decimal_alpha . $decimal_point2;
        $lcc_class =~ s/ /0/g;

        @lccclass = split(//, $lcc_class);
        $lcc_range = $alphanumarray{$lccclass[0]} . $alphanumarray{$lccclass[1]} . $alphanumarray{$lccclass[2]} . $lccclass[3] . $lccclass[4] . $lccclass[5] . $lccclass[6] . $lccclass[7] . $lccclass[8] . $lccclass[9] . $lccclass[10] . $lccclass[11] . $alphanumarray{$lccclass[12]} . $lccclass[13] . $lccclass[14] . $lccclass[15] . $lccclass[16] . $lccclass[17] . $lccclass[18];
        uniqueness_diacritics('lcc_range', '', $lcc_range);

#print $lc_callnumber . "\n" . $lcc_class . "\n" . $lcc_range . "\n\n";


        $lcc_group = substr($lcc_range, 0, 2);
        $comparison_lcc_range = substr($lcc_range, 0, 10) . "." . substr($lcc_range, 10);
        while ($c <= $#{$alllcclasses_array[$lcc_group]} ) {
            if($alllcclasses_array[$lcc_group][$c][1] <= $comparison_lcc_range) {
                if($alllcclasses_array[$lcc_group][$c][2] >= $comparison_lcc_range) {
                 push(@these_lc_class_labels, $alllcclasses_array[$lcc_group][$c][0]);
                }
            } else {
                $c = $#{$alllcclasses_array[$lcc_group]};
            }
            $c++;
        }
        $this_lc_class_label = join("\\n", @these_lc_class_labels);
        $query_insert{'lc_class_label'} = $this_lc_class_label;
        undef(@these_lc_class_labels);
    }

}

foreach $f052 ($record->field('052')) {
    foreach $f052_subfields ($f052->subfields()) {
        ($f052code, $f052data ) = @$f052_subfields;

        if($f052code =~ /a/) {
            foreach $i (0, 2) {
                $event_place_code = 'G' . $f052data;
                $this_event_place = $event_places_hash{$event_place_code};
                if($this_event_place ne '') {
                    uniqueness('geographics_facet', 'GENERIC\\t', $this_event_place);
                }
            }
        } elsif($f052code =~ /b/) {
            foreach $i (0, 2) {
                $event_subplace_code = $event_place_code . '.' . $f052data;
                $this_event_place = $event_places_hash{$event_subplace_code};
                if($this_event_place ne '') {
                    uniqueness('geographics_facet', 'GENERIC\\t', $this_event_place);
                }
            }
        }
    }
}

foreach $tracing ($record->field('[14]..|6[01349].|65[015]|7[013].|75[013-9]|8[0-4].|85[0-57-9]|8[6-9].')) {
    $this_tag = $tracing->tag();
    $indicator1 = $tracing->indicator(1);
    $indicator2 = $tracing->indicator(2);
    $raw_tracing = '';
    $this_volume = '';
    $this_role = '';
    $index_name = '';
    $this_thesaurus = '';
    $institution_to_which_this_applies = '';

        $tracing_sequence++;
        $tracing_subfields_count = 0;

    foreach $tracing_subfields ($tracing->subfields()) {
        ($tracing_code, $tracing_data ) = @$tracing_subfields;
        if($tracing_code !~ /[e245vw]/) {
            $tracing_subfields_count++;
            $raw_tracing .= '|' . $tracing_code . $tracing_data;
        } elsif($tracing_code =~ /[e4]/) {
            $this_role = $tracing_data;
            $raw_tracing =~ s/([\.,:;\/]$| $)//g;
        } elsif($tracing_code =~ /[2]/) {
            $this_thesaurus = $tracing_data;
            $raw_tracing =~ s/([\.,:;\/]$| $)//g;
        } elsif($tracing_code =~ /[5]/) {
            $institution_to_which_this_applies = $tracing_data;
            $raw_tracing =~ s/([\.,:;\/]$| $)//g;
        } elsif($tracing_code =~ /v/) {
            if($this_tag =~ /6../) {
                $tracing_subfields_count++;
                $raw_tracing .= '|' . $tracing_code . $tracing_data;
            } else {
                $this_volume = $tracing_data;
                $raw_tracing =~ s/([\.,:;\/]$| $)//g;
            }
        }
    }

    if($this_tag =~ /[1478]00/) {
        $index_name = 'authors_personal';
    } elsif($this_tag =~ /[17]10/) {
        $index_name = 'authors_corporate';
    } elsif($this_tag =~ /[17]11/) {
        $index_name = 'authors_conferences';
    } elsif($this_tag =~ /440|81[01]|830/) {
        $index_name = 'series_traced';
    } elsif($this_tag =~ /4[019]/) {
        $index_name = 'series_untraced';
    } elsif($this_tag =~ /[17]30/) {
        $index_name = 'titles_uniform';
    } elsif($this_tag =~ /600/) {
        $index_name = 'subjects_personal';
    } elsif($this_tag =~ /610/) {
        $index_name = 'subjects_corporate';
    } elsif($this_tag =~ /611/) {
        $index_name = 'subjects_conferences';
    } elsif($this_tag =~ /630/) {
        $index_name = 'subjects_titles';
    } elsif($this_tag =~ /648/) {
        $index_name = 'subjects_chronological';
    } elsif($this_tag =~ /650/) {
        $index_name = 'subjects_topical';
    } elsif($this_tag =~ /651/) {
        $index_name = 'subjects_geographic';
    } elsif($this_tag =~ /655/) {
        $index_name = 'subjects_genre';
    } elsif($this_tag =~ /790/) {
        $index_name = 'authors_personal';
    } elsif($this_tag =~ /791/) {
        $index_name = 'authors_corporate';
    }



$raw_tracing =~ s/[\.]*\|t/|t/g;
($raw_tracing_normalized = $raw_tracing) =~ s/\|[vx-z]/--/g;
$raw_tracing_normalized =~ s/\|t/\\t/;
$raw_tracing_normalized =~ s/\|[a]//g;
$raw_tracing_normalized =~ s/\|[b-suw]/ /g;

uniqueness_diacritics('raw_tracing', '', $tracing_sequence . "\t" . $this_tag . "\t" . $indicator1 . "\t" . $indicator2 . "\t" . $raw_tracing_normalized . "\t" . $index_name . "\t" . $this_role . "\t" . $this_thesaurus . "\t" . $institution_to_which_this_applies . "\t" . $raw_tracing . "\t" . $this_volume . "\t" . $oclc_arn);

}

$mainheading = '';

foreach $f1XX ($record->field('100|110|111')) {
    $this_tag = $f1XX->tag();
    $roles = '';

    if($this_tag =~ /100/) {
        $hash_tag = 'authors_personal';
        $codes = '[a-dgjq]';
        $relators = '[e4]';
    } elsif($this_tag =~ /110/) {
        $hash_tag = 'authors_corporate';
        $codes = '[a-dgjknpqu]';
        $relators = '[e4]';
    } elsif($this_tag =~ /111/) {
        $hash_tag = 'authors_conferences';
        $codes = '[ac-enpu]';
        $relators = '[4]';
    }


    foreach $f1XX_subfields ($f1XX->subfields()) {
    ($f1XXcode, $f1XXdata ) = @$f1XX_subfields;
        if($f1XXcode =~ /$codes/) {
            if($f1XXcode =~ /a/) {
                $mainheading .= $f1XXdata;
            } else {
                $mainheading .= ' ' . $f1XXdata;
            }
        } elsif($f1XXcode =~ /$relators/) {
          ($this_mainheading = $mainheading) =~ s/[\s]*[\.,]*[\s]*$//g;
            if($f1XXcode =~ /4/) {
                if(exists($relators_hash{$f1XXdata})) {
                    uniqueness_diacritics('roles', $this_mainheading, $relators_hash{$f1XXdata});
                    $roles = $relators_hash{$f1XXdata};
                } else {
                    uniqueness_diacritics('roles', $this_mainheading, $f1XXdata);
                    $roles = $f1XXdata;
                }
            } else {
                uniqueness_diacritics('roles', $this_mainheading, $f1XXdata);
                $roles = $f1XXdata;
            }
        }
    }

    uniqueness_diacritics('mainheading', $mainheading, $roles);
    $mainheading =~ s/[\s]*[\.,]*[\s]*$//g;
    uniqueness_diacritics($hash_tag, $mainheading, '');
}

foreach $f130 ($record->field('130|24[023]')) {
    $this_tag = $f130->tag();
    $this_work_title = '';
    $uniformtitle_extras = '';

    foreach $f130_subfields ($f130->subfields()) {
    ($f130code, $f130data ) = @$f130_subfields;
            if($f130code =~ /[adgnp]/) {
                $this_work_title .= $f130data;
            } elsif($f130code =~ /f/) {
                #DATE OF WORK -- NOT NEEDED FOR worktitles
                $uniformtitle_extras .= ' ' . $f130data;
            } elsif($f130code =~ /l/) {
                $f130data =~ s/(^ |[\.,:;\/]$| $)//g;
                uniqueness_diacritics('languages', ' ', $f130data);
                $uniformtitle_extras .= ' ' . $f130data;
            } elsif($f130code =~ /k/) {
                $f130data =~ s/(^ |[\.,:;\/]$| $)//g;
                uniqueness_diacritics('genres_facet', 'GENERIC\\t', $f130data);
                $uniformtitle_extras .= ' ' . $f130data;
            } elsif($f130code =~ /m/) {
                $this_work_title .= ' ' . $f130data;
                uniqueness_diacritics('instrumentations_facet', 'UNFORMATTED', $f130data);
                $uniformtitle_extras .= ' ' . $f130data;
            } elsif($f130code =~ /[ors]/) {
                #ARRANGED STATEMENT -- NOT NEEDED FOR worktitles
                #KEY FOR MUSIC -- NOT NEEDED FOR worktitles
                #VERSION -- NOT NEEDED FOR worktitles
                $uniformtitle_extras .= ' ' . $f130data;
            }
    }
    
    $this_work_title =~ s/^ |([\.,:;\/]|[\.,:;\/] | )$//g;
    $this_work_title = ucfirst($this_work_title);
    uniqueness_diacritics('worktitles', "$mainheading ", $this_work_title);
    uniqueness_diacritics('uniformtitle', $this_work_title, $uniformtitle_extras);
    uniqueness_diacritics('titles_uniform', '', $this_work_title);
}

foreach $f2XX ($record->field('210|222')) {
    uniqueness_diacritics('titles_alternate', '', $f2XX->as_string);
    uniqueness_diacritics('tracings_titles_alternate', ' ', $f2XX->as_string);
}

foreach $f245 ($record->field('245')) {
    $this_subtitle = '';
    $this_work_title = '';
    $this_statement_of_responsibility = '';
    $subfield_a_count = 0;

    $skip = $f245->indicator(2);

    uniqueness_diacritics('maintitle', '', $f245->as_string);
    uniqueness_diacritics('maintitle_article', '', substr($f245->as_string, 0, $skip));

    foreach $f245_subfields ($f245->subfields()) {
        ($f245code, $f245data ) = @$f245_subfields;
        if(($f245code =~ /[a]/) && ($subfield_a_count == 0)){
            uniqueness_diacritics('title_proper', '', substr(($f245data), $skip) . ' ');
            $this_work_title = substr(($f245data), $skip) . ' ';
            $subfield_a_count++;
        } elsif($f245code =~ /[h]/) {
            #THE gmd HAS BEEN ASSIGNED EARLIER IN THIS SCRIPT
        } elsif($f245code =~ /[bfgknps]/) {
            $this_subtitle .= $f245data . ' ';
            if($f245code =~ /[nps]/) {
                $this_work_title .= $f245data . ' ';
            }
        } elsif($f245code =~ /[c]/) {
            $this_statement_of_responsibility .= $f245data . ' ';
        }

    }

    uniqueness_diacritics('subtitle', '', $this_subtitle);
    uniqueness_diacritics('statement_of_responsibility', '', $this_statement_of_responsibility);
    if($query_insert{'worktitles'} eq '') {
        $this_work_title =~ s/([\.,:;\/]|[\.,:;\/] | )$//g;
        $this_work_title = ucfirst($this_work_title);
        uniqueness_diacritics('worktitles', "$mainheading ", $this_work_title);
    }
}


foreach $f246 ($record->field('246')) {
    $this_title = '';
    $this_label = '';
    
    foreach $f246_subfields ($f246->subfields()) {
        ($f246code, $f246data ) = @$f246_subfields;
        if($f246code =~ /a/) {
            $this_title .= $f246data;
        } elsif($f246code =~ /[bfgnp]/) {
            $this_title .= ' ' . $f246data;
        } elsif($f246code =~ /[6]/) {
            #uniqueness_diacritics('unaccounted_marc_fields', '', '246 has vernacular form of tracing' . $f246data);
        } elsif($f246code =~ /[i]/) {
            $this_label = $record->field('246')->subfield('i');
        }
    } 
        if(($this_label eq '') && ($f246_type_hash{$f246->indicator(2)})){
            $this_label = $f246_type_hash{$f246->indicator(2)} . ': ';
        }
        
   uniqueness_diacritics('titles_alternate', '', $this_title);
   uniqueness_diacritics('tracings_titles_alternate', "$this_label ", $this_title);

}

foreach $f247 ($record->field('247')) {
    $this_title = '';
    $this_volumes = '';
    $this_label = 'Title varies: ';

    foreach $f247_subfields ($f247->subfields()) {
        ($f247code, $f247data ) = @$f247_subfields;
        if($f247code =~ /a/) {
            $this_title .= $f247data;
        } elsif($f247code =~ /[bgnp]/) {
            $this_title .= ' ' . $f247data;
        } elsif($f247code =~ /[f]/) {
            $this_volumes .= ' ' . $f247data;
        } elsif($f247code =~ /[6]/) {
            # uniqueness_diacritics('unaccounted_marc_fields', '', '247 has vernacular form of tracing');
        } elsif($f247code =~ /[x]/) {
            uniqueness('isn', '', $f247data);
        }
    }
        if(($this_label eq '') && ($f247_type_hash{$f247->indicator(2)})){
            $this_label = $f247_type_hash{$f247->indicator(2)} . ': ';
        }

   uniqueness_diacritics('titles_analytic', '', $this_title);
   uniqueness_diacritics('tracings_titles_analytic', $this_label, $this_title . $this_volumes);

}


foreach $f25X ($record->field('25[04]')) {
    uniqueness_diacritics('edition', '', $f25X->as_string);
}

foreach $f25XX ($record->field('25[5-8]')) {
    $this_tag = $f25XX->tag();
    $this_note = $f25XX->as_string;
    $this_label = '';

    if($this_tag =~ /255/) {
        $this_label = 'Cartographic Mathematical Data: ';
    } elsif($this_tag =~ /256/) {
        $this_label = 'Computer File Characteristics: ';
    } elsif($this_tag =~ /257/) {
        $this_label = 'Archival film produced in ';
    } elsif($this_tag =~ /258/) {
        $this_label = 'Philatelic Issue Data: ';
    }

    uniqueness_diacritics('notes_public', "$this_label ", $this_note);
}

foreach $f26X ($record->field('26[012]')) {
    uniqueness_diacritics('imprint', '', $f26X->as_string);
    $this_pubplace = '';

    foreach $f26X_subfields ($f26X->subfields()) {
        ($f26Xcode, $f26Xdata ) = @$f26X_subfields;

        if($f26Xcode =~ /a/) {
            ($pubplace = $f26Xdata) =~ s/([,:;\/]$|\[|\]|[\s]*$)//g;
            $pubplace =~ s/i\.e\.,//g;
            ($this_pubplace = $pubplace) =~ s/[\s]*$/./;
            uniqueness_diacritics('pubplace', '', $this_pubplace);

            @parts = split(/,/, $pubplace);
            if($#parts > 0) {
                $pubplace_expanded = $parts[0];
                for($i = 1; $i <= $#parts; $i++) {
                    $parts[$i] =~ s/^[\s]*|[\s]$//g;
                    if($states_hash{$parts[$i]} ne '') {
                        $pubplace_expanded .= ' (' . $states_hash{$parts[$i]} . ') ';
                    } else {
                        $pubplace_expanded .= ' (' . $parts[$i] . ') ';
                    }
                }
            } else {
                if($pubplace =~ /\(([a-zA-Z.]*)\)/) {
                    $state = $1;
                    if($states_hash{$state} ne '') {
                        $pubplace =~ s/$state/$states_hash{$state}/;
                    }
                }
                $pubplace =~ s/^[\s]*|[\s]*$//g;
                $pubplace_expanded = $pubplace;
            }
            uniqueness_diacritics('pubplace_facet', '', $pubplace_expanded);
        } elsif($f26Xcode =~ /b/) {
            $f26Xdata =~ s/[,]$//g;
            $publisher = $f26Xdata;
            $publisher =~ s/i\.e\.,//g;
            uniqueness_diacritics('publisher', '', $publisher);
        }
    }
}

foreach $f30X ($record->field('30[05]')) {
    $indicator1 = $f30X->indicator(1);
    $indicator2 = $f30X->indicator(2);
    $this_tag = $f30X->tag();
    $this_label = '';
    $total_pages = '';
    $total_roman = '';
    $these_pages = '';
    undef(@roman);

    foreach $f30X_subfields ($f30X->subfields()) {
        ($f30Xcode, $f30Xdata ) = @$f30X_subfields;
        if($f30Xcode =~ /a/) {
            $this_label = $f30Xdata;
            $f30Xdata_string = $f30Xdata;
            $f30Xdata_string =~ s/[\[\]\(\)]//g;
            $f30Xdata_string =~ s/[0-9]*[ ]*p\.[ ]*l\.|[0-9]*[ ]*[lv]\.//g;
            ($roman_numeral_string = $f30Xdata_string) =~ s/, [0-9]*, |, [0-9]* | [0-9]*, | [0-9]* / /g;

            foreach $roman_numeral ($roman_numeral_string =~ /((?:[ivxlcdm]*)*[,]*[ ]*[ivxlcdm]*)[ ]*[,]*[ ]*p(?:\.|\s|$)/g) {

    foreach $number ($roman_numeral =~ /([ivxlcdm]*)/g) {
#print "number: " . $number . "\n";

                $roman_numeral = arabic($number);

#print "roman_numeral: " . $roman_numeral . "\n";

            $total_roman += $roman_numeral;
            }
            }

            ($arabic_numeral_string = $f30Xdata_string) =~ s/, [ivxlcdm]*, |, [ivxlcdm]* | [ivxlcdm]*, | [ivxlcdm]* / /g;
            foreach $this_page ($arabic_numeral_string =~ /((?:[0-9]*)*[,]*[ ]*[0-9]*)[ ]*[,]*[ ]*p(?:\.|\s|$)/g) {

                $these_pages .= $this_page . ' ';
#print "\n" . $this_page . "\n";
                foreach $number ($this_page =~ /([0-9]*)/g) {
#print $number . "\n";
                    $total_pages += $number;
                }
            }
            if ($f30Xdata_string =~ /^p.[ ]*([0-9]*)-([0-9]*)/) {
                $these_pages .= $1 . "-" . $2;
                $this_page = ($2 - $1) + 1;
                $total_pages += $this_page;
            } elsif ($f30Xdata_string =~ /^p.[ ]*([0-9]*) /) {
                $these_pages .= $1;
                $this_page = 1;
                $total_pages += $this_page;
            }

$query_insert{'roman_pages'} += $total_roman;
$query_insert{'pages'} += $total_pages;
#            uniqueness('pages', "$this_label " , "$f30Xdata_string\\t$total_roman\\t($these_pages)\\t$total_pages");
    undef($total_pages);
    undef($total_roman);
    undef($these_pages);

        }
    }

    uniqueness_diacritics('extent', '', $f30X->as_string);
}

foreach $f306 ($record->field('306')) {
    $indicator1 = $f306->indicator(1);
    $indicator2 = $f306->indicator(2);
    $this_tag = $f306->tag();

    foreach $f306_subfields ($f306->subfields()) {
        ($f306code, $f306data ) = @$f306_subfields;
        if($f306code =~ /a/) {
            $hours2seconds = substr($f306data, 0, 2) * 3600;
            $minutes2seconds = substr($f306data, 2, 2) * 60;
            $this_duration = $hours2seconds + $minutes2seconds + substr($f306data, 4, 2);
            $total_durations += $this_duration;
        }
    }
    uniqueness('duration', '', $total_durations);
    undef($total_durations);
}

foreach $f3XX ($record->field('307|310|321|34[023]|35[12]|362')) {
    $this_tag = $f3XX->tag();
    $this_label = '';

if($this_tag =~ /307/) {
    $this_label = 'Hours: ';
} elsif($this_tag =~ /340/) {
    $this_label = 'Physical Medium: ';
} elsif($this_tag =~ /342/) {
    $this_label = 'Geospatial Reference Data: ';
} elsif($this_tag =~ /343/) {
    $this_label = 'Planar Coordinate Data: ';
}

    uniqueness_diacritics('notes_public', "$this_label ", $f3XX->as_string);
}

foreach $f400 ($record->field('400|41[01]')) {
    $this_tag = $f400->tag();
    $this_series_title = '';
    $this_series_volume = '';
    $this_series_hash = '';

    if($this_tag =~ /400/) {
        $hash_tag = 'authors_personal';
        $codes = '[a-djqu]';
        $title_parts = '[f-hk-pr-t]';
        $relators = '[e4]';
    } elsif($this_tag =~ /410/) {
        $hash_tag = 'authors_corporate';
        $codes = '[a-du]';
        $title_parts = '[fgk-pr-t]';
        $relators = '[e4]';
    } elsif($this_tag =~ /411/) {
        $hash_tag = 'authors_conferences';
        $codes = '[ac-equ]';
        $title_parts = '[f-hklnpst]';
        $relators = '[4]';
    }


    foreach $f400_subfields ($f400->subfields()) {
    ($f400code, $f400data ) = @$f400_subfields;
        if($f400code =~ /$codes/) {
            if($f400code =~ /a/) {
                $this_series_hash .= $f400data;
            } else {
                $this_series_hash .= ' ' . $f400data;
            }
        } elsif($f400code =~ /$relators/) {
            $this_series_hash =~ s/[\.,]$//g;
            if($f400code =~ /4/) {
                if(exists($relators_hash{$f400data})) {
                    uniqueness_diacritics('roles', $this_series_hash, $relators_hash{$f400data});
                } else {
                    uniqueness_diacritics('roles', $this_series_hash, $f400data);
                }
            } else {
                uniqueness_diacritics('roles', $this_series_hash, $f400data);
            }
        } elsif($f400code =~ /$title_parts/) {
            if($f400code =~ /l/) {
                $f400data =~ s/(^ |[\.,:;\/]$| $)//g;
                uniqueness_diacritics('languages', 'Primary series: ', $f400data);
            } elsif($f400code =~ /m/) {
                $this_series_title .= $f400data . ' ';
                uniqueness_diacritics('instrumentations_facet', 'UNFORMATTED', $f400data);
            } else {
                $this_series_title .= $f400data . ' ';
            }
        } elsif($f400code =~ /[v]/) {
                $this_series_volume .= $f400data . ' ';
        } else {
            $this_series_title .= $f400data . ' ';
        }
    }

    $this_series_hash =~ s/[\s]*[\.,]*[\s]*$//g;
    $this_series_title =~ s/([\s]*[\.,:;\/]*[\s]*|[\s]*[\.,:;\/]*[\s]* |[\s]*)$//g;
    uniqueness_diacritics('series_traced', $this_series_hash . '\\t' . $this_series_title, $this_series_volume);
    uniqueness_diacritics('titles_series', '', $this_series_title);
    uniqueness_diacritics($hash_tag, '', $this_series_hash);
}

foreach $f4XX ($record->field('440|490')) {
    $indicator1 = $f4XX->indicator(1);
    $indicator2 = $f4XX->indicator(2);
    $this_tag = $f4XX->tag();
    $this_series_title = '';
    $this_series_volume = '';

    if($this_tag =~ /490/) {
        $type_of_series = 'series_untraced';
    } else {
        $type_of_series = 'series_traced';
    }
    
    foreach $f4XX_subfields ($f4XX->subfields()) {
        ($f4XXcode, $f4XXdata ) = @$f4XX_subfields;

        if($f4XXcode =~ /[a]/) {
            if($this_tag =~ /440/) {
                $this_series_title .= substr(($f4XXdata), $indicator2);
            } else {
                $this_series_title .= $f4XXdata;
            }
        } elsif($f4XXcode =~ /[np]/) {
            $this_series_title .= ' ' . $f4XXdata;
        } elsif($f4XXcode =~ /[x]/) {
            if($f4XXdata =~ /([0-9xX]{4}-[0-9xX]{4}).*/) {
                $this_isn = $1;
            } else {
                $this_isn = $f4XXdata;
            }
            uniqueness('isn', 'ISSN: ' . '\\t', $this_isn);
        } elsif($f4XXcode =~ /[v]/) {
            $this_series_volume .= $f4XXdata;
        }

    }

    $this_series_title =~ s/([\s]*[\.,:;\/]*[\s]*|[\s]*[\.,:;\/]*[\s]* |[\s]*)$//g;

    uniqueness_diacritics($type_of_series, '\\t' . $this_series_title, $this_series_volume);
    uniqueness_diacritics('titles_series', '', $this_series_title);
}

foreach $f5XX ($record->field('5..')) {
    $indicator1 = $f5XX->indicator(1);
    $indicator2 = $f5XX->indicator(2);
    $this_tag = $f5XX->tag;
    $this_label = '';
    $this_note = '';

    $this_note = $f5XX->subfield('a');

    if($this_tag =~ '504') {
        if($f5XX->subfield('b')) {
            $this_note .= ' (Number of references: ' . $f5XX->subfield('b') . ')';
        }

    } elsif($this_tag =~ '505') {
        if($indicator1 =~ /0/) {
            $this_label = 'Contents: ';
        } elsif($indicator1 =~ /1/) {
            $this_label = 'Incomplete contents: ';
        } elsif($indicator1 =~ /2/) {
            $this_label = 'Partial contents: ';
        } elsif($indicator1 =~ /8/) {
            $this_label = '';
        }

        $this_note = $f5XX->as_string;

#                foreach $f5XX_subfields ($f5XX->subfields()) {
#                    ($f5XXcode, $f5XXdata ) = @$f5XX_subfields;
#                        if($f5XXcode =~ /[a]/) {
#                           @titles_auths = split(/--/, $f5XXdata);
#                           for($i = 0; $i <= $#titles_auths; $i++) {
#                               @all_titles = split(/\//, $titles_auths[$i]);
#                            uniqueness_diacritics('titles_analytic', '', $all_titles[0]);
#
#                            @all_authors = split(/ and |;/, $all_titles[1]);
#                               for($j = 0; $j <= $#all_authors; $j++) {
#                                   @authors = split(/ /, $all_authors[$j]);
#                                       $last_element = $#authors;
#                                       $surname = $authors[$last_element];
#                                       $given_names = '';
#                                           for($k = 0; $k < $#authors; $k++) {
#                                               $given_names .= $authors[$k] . ' ';
#                                           }
#                                $this_author = $surname . ', ' . $given_names;
#                                    uniqueness_diacritics('authors_uncontrolled', '', $this_author);
#                               }
#                           }
#                        } elsif($f5XXcode =~/[gu]/) {
#                            #miscellaneous information and URI
#                        } elsif($f5XXcode =~/[r]/) {
#                           @all_authors = split(/and|;/, $f5XXdata);
#                           for($j = 0; $j <= $#all_authors; $j++) {
#                               @authors = split(/ /, $all_authors[$i]);
#                                   $last_element = $#authors;
#                                   $surname = $authors[$last_element];
#                                   $given_names = '';
#                                   for($k = 0; $k < $#authors; $k++) {
#                                       $given_names .= $authors[$k] . ' ';
#                                   }
#
#                           uniqueness_diacritics('authors_uncontrolled', '', $surname . ', ' . $given_names);
#                           }
#
#                        } elsif($f5XXcode =~/[t]/) {
#                            $f5XXdata =~ s/\/$//g;
#                            uniqueness_diacritics('titles_analytic', '', $f5XXdata);
#                    }
#                }

    } elsif($this_tag =~ '508') {
        $this_label = 'Credits: ';
        
#        @credits = split(/;/, $f5XX->subfield('a'));
#            for($i = 0; $i <= $#credits; $i++) {
#                @credits_split = split(/,/, $credits[$i]);
#                    $role_name = $credits_split[0];
#                    for($j = 0; $j <= $#credits_split; $j++) {
#                        @directors = split(/ /, $credits_split[$j]);
#                            $last_element = $#directors;
#                            $surname = $directors[$last_element];
#                            $given_names = '';
#                            for($k = 0; $k < $#directors; $k++) {
#                                $given_names .= $directors[$k] . ' ';
#                            }
#               
#                    }    
#                    $contibutor = $surname . ', ' . $given_names;
#                    uniqueness_diacritics('roles', $role_name, $contibutor);
#                    uniqueness_diacritics('authors_uncontrolled', '', $contibutor);
#            }

    } elsif($this_tag =~ '510') {
        if($indicator1 =~ /0/) {
            $this_label = 'Indexed by: ';
        } elsif($indicator1 =~ /1/) {
            $this_label = 'Indexed in its entirety by: ';
        } elsif($indicator1 =~ /2/) {
            $this_label = 'Indexed selectively by: ';
        } elsif($indicator1 =~ /[3-4]/) {
            $this_label = 'References: ';
        }

    } elsif($this_tag =~ '511') {
        if($indicator1 =~ /1/) {
            $this_label = 'Cast: ';
        }

#        @cast = split(/;/, $f5XX->subfield('a'));
#        for($i = 0; $i <= $#cast; $i++) {
#            @cast_split = split(/,/, $cast[$i]);
#            for($j = 0; $j <= $#cast_split; $j++) {
#                if($cast_split[$j] =~ /(.*):(.*)/) {
#                    $this_role = $1;
#                    $cast_split[$j] = $2;
#                } else {
#                    $this_role = 'Cast';
#                }
#                @actors = split(/ /, $cast_split[$j]);
#                $last_element = $#actors;
#                $surname = $actors[$last_element];
#                $given_names = '';
#                for($k = 0; $k < $#actors; $k++) {
#                    $given_names .= $actors[$k] . ' ';
#                }
#                $contibutor = $surname . ', ' . $given_names;
#                uniqueness_diacritics('roles', $role_name, $contibutor);
#                uniqueness_diacritics('authors_uncontrolled', '', $contibutor);
#
#            }
#        }
#        undef($this_role);

    } elsif($this_tag =~ '516') {
        if($indicator1 =~ /[ ]/) {
            $this_label = 'Type of file: ';
        }

    } elsif($this_tag =~ '518') {
        foreach $i ($this_note =~ /\b[12][89][0-9][0-9]-[12][89][0-9][0-9]\b|\b[12][89][0-9][0-9]\b/g) {
            dates_expansion('event_dates_facet', 'GENERIC', $i);
        }
    
    } elsif($this_tag =~ '520') {
        if($indicator1 =~ /[ ]/) {
            $this_label = 'Summary: ';
        } elsif($indicator1 =~ /0/) {
            $this_label = 'Subject: ';
        } elsif($indicator1 =~ /1/) {
            $this_label = 'Review: ';
        } elsif($indicator1 =~ /2/) {
            $this_label = 'Scope and content: ';
        } elsif($indicator1 =~ /3/) {
            $this_label = 'Abstract: ';
        } elsif($indicator1 =~ /8/) {
            $this_label = '';
        }

    } elsif($this_tag =~ '521') {
        $this_label = '';

        if($indicator1 =~ /0/) {
            $this_label = 'Reading grade level: ';
        } elsif($indicator1 =~ /1/) {
            $this_label = 'Interest age level: ';
        } elsif($indicator1 =~ /2/) {
            $this_label = 'Interest grade level: ';
        } elsif($indicator1 =~ /3/) {
            $this_label = 'Special audience characteristics: ';
        } elsif($indicator1 =~ /4/) {
            $this_label = 'Motivation/interest level: ';
        }

        foreach $subfielda ($f5XX->subfield('a')) {
            $subfielda =~ s/^'|'$//g;
            if($query_insert{'audience'} =~ /Unspecified/) {
                $query_insert{'audience'} = $this_label . $subfielda;
            } else {
                uniqueness_diacritics('audience', '', $this_label . $subfielda);
            }
        }

    } elsif($this_tag =~ '522') {
        if($indicator1 =~ /[ ]/) {
            $this_label = 'Geographic coverage: ';
        }

    } elsif($this_tag =~ '524') {
        if($indicator1 =~ /[ ]/) {
            $this_label = 'Cite as: ';
        }
    
    } elsif($this_tag =~ '526|590') {
        $this_note = ''; # BECAUSE THERE ARE POSSIBLE NON-PUBLIC NOTES, THIS TIME this_note HAS TO BE CREATED HERE
        foreach $f5XX_subfields ($f5XX->subfields()) {
            ($f5XXcode, $f5XXdata ) = @$f5XX_subfields;

            if($f5XXcode !~ /[x]/) { # SUBFIELD x IS FOR NON-PUBLIC NOTES
                $this_note .= ' ' . $f5XXdata;
            } else {
                uniqueness_diacritics('notes_nonpublic', '', $f5XXdata);
            }

        }

        if($indicator1 =~ /0/) {
            $this_label = 'Reading program: ';
        }

    } elsif($this_tag =~ '533') {
        $this_label = 'Reproduction notes: ';
        $this_note = $f5XX->as_string;

    } elsif($this_tag =~ '535') {
        if($indicator1 =~ /1/) {
            $this_label = 'Holder of originals: ';
        } elsif($indicator1 =~ /2/) {
            $this_label = 'Holder of duplicates: ';
        }

    } elsif($this_tag =~ '541') {
        $this_note = ''; # BECAUSE THERE ARE POSSIBLE NON-PUBLIC NOTES, THIS TIME this_note HAS TO BE CREATED HERE
        if($indicator1 ne '1') {
            foreach $f5XX_subfields ($f5XX->subfields()) {
                ($f5XXcode, $f5XXdata ) = @$f5XX_subfields;

                if($f5XXcode !~ /[h]/) { # SUBFIELD h IS FOR THE PURCHASE PRICE
                    $this_note .= ' ' . $f5XXdata;
                } else {
                    uniqueness_diacritics('notes_nonpublic', '', $f5XXdata);
                }
            }

        } else {
            uniqueness_diacritics('notes_nonpublic', '', $f5XX->as_string);
        }

    } elsif($this_tag =~ '544') {
        if($indicator1 =~ /0/) {
            $this_label = 'Associated materials: ';
        } elsif($indicator1 =~ /1/) {
            $this_label = 'Related materials: ';
        }

    } elsif($this_tag =~ '545') {
        if($indicator1 =~ /0/) {
            $this_label = 'Biographical sketch: ';
        } elsif($indicator1 =~ /1/) {
            $this_label = 'Administrative history: ';
        }

    } elsif($this_tag =~ '546') {
        foreach $f5XX_subfields ($f5XX->subfields()) {
            ($f5XXcode, $f5XXdata ) = @$f5XX_subfields;

            if($f5XXcode =~ /[a]/) {

                foreach $languages_spelled_out (values %languages_hash) {
                    if($f5XXdata =~ /\Q$languages_spelled_out\E/) {
                        if(($f5XXdata =~ /Translat(?:ion|ed)(?:s)* (?:of|from).*\Q$languages_spelled_out\E.*(?:in|into)*/i) && ($f5XXdata !~ /(?:in|into).*\Q$languages_spelled_out\E/i)) {
                            uniqueness('languages', '\\tTranslation from ', $languages_spelled_out);
                        } else {
                            uniqueness('languages', '\\t', $languages_spelled_out);
                        }
                    }

                    if($f5XXdata =~ /dubbed \Q$languages_spelled_out\E|\Q$languages_spelled_out\E-dubbed|\Q$languages_spelled_out\E(?:[a-z0-9 ]*or |,| )[0-9a-z ]*dubbed|dubbed[0-9a-z ]*in(?:[a-z0-9 ]*or | )\Q$languages_spelled_out\E|dubbed[0-9a-z ]*into \Q$languages_spelled_out\E/i) {
                        uniqueness('languages', 'Dubbed in: \\t', $languages_spelled_out);
                    }
                }
            } elsif($f5XXcode =~ /[b]/) {
                uniqueness_diacritics('languages', ' ', $f5XXdata);
            }
        }

    } elsif($this_tag =~ '555') {
        if($indicator1 =~ /[ ]/) {
            $this_label = 'Indexes: ';
        } elsif($indicator1 =~ /1/) {
            $this_label = 'Finding aids: ';
        }

    } elsif($this_tag =~ '556') {
        if($indicator1 =~ /[ ]/) {
            $this_label = 'Documentation: ';
        }

    } elsif($this_tag =~ '561') {
        $this_note = ''; # BECAUSE THERE ARE POSSIBLE NON-PUBLIC NOTES, THIS TIME this_note HAS TO BE CREATED HERE
        if($indicator1 ne '1') {
            $this_note = $f5XX->as_string;
        } else {
           uniqueness_diacritics('notes_nonpublic', '', $f5XX->as_string);
        }

    } elsif($this_tag =~ '563') {
        $this_label = 'Binding: ';

    } elsif($this_tag =~ '565') {
        if($indicator1 =~ /[ ]/) {
            $this_label = 'File size: ';
        } elsif($indicator1 eq '0') {
            $this_label = 'Case file characteristics: ';
        }

    } elsif($this_tag =~ '567') {
        if($indicator1 =~ /[ ]/) {
            $this_label = 'Methodology: ';
        }

    } elsif($this_tag =~ '581') {
        if($indicator1 =~ /[ ]/) {
            $this_label = 'Publications: ';
        }

    } elsif($this_tag =~ '583') {
        $this_note = ''; # BECAUSE THERE ARE POSSIBLE NON-PUBLIC NOTES, THIS TIME this_note HAS TO BE CREATED HERE
        if($indicator1 ne '1') {
            $this_label = 'Action note: ';
            foreach $f5XX_subfields ($f5XX->subfields()) {
                ($f5XXcode, $f5XXdata ) = @$f5XX_subfields;

                if($f5XXcode !~ /[x]/) { # SUBFIELD x IS FOR NON-PUBLIC NOTES
                    $this_note .= ' ' . $f5XXdata;
                } else {
                    uniqueness_diacritics('notes_nonpublic', '', $f5XXdata);
                }
            }

        } else {
            uniqueness_diacritics('notes_nonpublic', '', $f5XX->as_string);
        }

    }

    uniqueness_diacritics('notes_public', "$this_label ", $this_note);
}

foreach $f6XX ($record->field('600|61[01]|648|65[013-8]|662|69[015-8]')) {
    $indicator1 = $f6XX->indicator(1);
    $indicator2 = $f6XX->indicator(2);
    $this_tag = $f6XX->tag;
    $this_hash = '';
    $thesaurus = '';
    $this_subjects_worktitles = '';
    $this_subjects_titles = '';

    if($this_tag =~ /600|61[01]|65[014-6]|69[6-8]/) {
        if(!$f6XX->subfield('2')) {
            $thesaurus = $thesauri_hash{$indicator2}; 
        } else {
            $thesaurus = uc($f6XX->subfield('2'));
        }
    }

    if($this_tag =~ /600/) {
        $hash_tag = 'subjects_personal';
        $codes = '[a-df-hj-vx-z]';
        $relators = '[e4]';
        $subparts = '[bcdgqu]';
    } elsif($this_tag =~ /610/) {
        $hash_tag = 'subjects_corporate';
        $codes = '[a-dfgk-pr-vx-z]';
        $relators = '[e4]';
        $subparts = '[bcdgqu]';
    } elsif($this_tag =~ /611/) {
        $hash_tag = 'subjects_conferences';
        $codes = '[ac-gklnps-vx-z]';
        $relators = '[4]';
        $subparts = '[bcdegqu]';
    } elsif($this_tag =~ /648|657/) {
        $hash_tag = 'subjects_chronological';
        $codes = '[avx-z]';
        $relators = '[ ]';
    } elsif($this_tag =~ /650/) {
        $hash_tag = 'subjects_topical';
        $codes = '[a-dvx-z]';
        $relators = '[e4]';
        $subparts = '[cd]';
    } elsif($this_tag =~ /651/) {
        $hash_tag = 'subjects_geographic';
        $codes = '[avx-z]';
        $relators = '[e4]';
    } elsif($this_tag =~ /653/) {
        $hash_tag = 'topics_facet';
        $codes = '[a]';
        $relators = '[ ]';
    } elsif($this_tag =~ /654/) {
        $hash_tag = 'topics_facet';
        $codes = '[a-cvx-z]';
        $relators = '[e4]';
    } elsif($this_tag =~ /655/) {
        $hash_tag = 'subjects_genre';
        $codes = '[a-cvx-z5]';
        $relators = '[ ]';
    } elsif($this_tag =~ /656/) {
        $hash_tag = 'subjects_corporate';
        $codes = '[akvx-z]';
        $relators = '[ ]';
    } elsif($this_tag =~ /658/) {
        $hash_tag = 'subjects_topical';
        $codes = '[a-d]';
        $relators = '[ ]';
    } elsif($this_tag =~ /662/) {
        $hash_tag = 'subjects_geographic';
        $codes = '[a-df-h]';
        $relators = '[e4]';
    } elsif($this_tag =~ /690/) {
        $hash_tag = 'subjects_topical';
        $codes = '[a-dvx-z9]';
        $relators = '[e]';
        $subparts = '[cd]';
    } elsif($this_tag =~ /691/) {
        $hash_tag = 'subjects_geographic';
        $codes = '[a-bvx-z9]';
        $relators = '[e]';
    } elsif($this_tag =~ /695/) {
        $hash_tag = 'local_callnumber';
        $codes = '[a-bef2]';
        $relators = '[ ]';
    } elsif($this_tag =~ /696/) {
        $hash_tag = 'subjects_personal';
        $codes = '[a-df-hj-vx-z9]';
        $relators = '[e4]';
        $subparts = '[bcdgqu]';
    } elsif($this_tag =~ /697/) {
        $hash_tag = 'subjects_corporate';
        $codes = '[a-df-hk-pr-vx-z9]';
        $relators = '[e4]';
        $subparts = '[bcdgqu]';
    } elsif($this_tag =~ /698/) {
        $hash_tag = 'subjects_conferences';
        $codes = '[ac-hklnps-vx-z9]';
        $relators = '[4]';
        $subparts = '[bcdegqu]';
    }

    foreach $f6XX_subfields ($f6XX->subfields()) {
        ($f6XXcode, $f6XXdata ) = @$f6XX_subfields;

        if($f6XXcode =~ /$codes/) {
            if($f6XXcode =~ /[a]/) {
                $this_subjects_worktitles .= ' ' . $f6XXdata;
                $this_hash .= $f6XXdata;
                $f6XXdata =~ s/\.$//;

                if($this_tag =~ /651|691/) {
                    uniqueness_diacritics('geographics_facet', $thesaurus, $f6XXdata);
                } elsif($this_tag =~ /655/) {
                    uniqueness_diacritics('genres_facet', $thesaurus, $f6XXdata);
                } elsif($this_tag =~ /650/) {
                    uniqueness_diacritics('topics_facet', $thesaurus, $f6XXdata);

#                    if($f6XXdata =~ /^($cardinals_keys)(?: |-)($cardinals_keys)(?: |-)($cardinals_keys)(?: $|$|, A.D)/i) {
#                        $hash_tag = 'subjects_chronological';
#                        $first = lc($1);
#                        $second = lc($2);
#                        $third = lc($3);
#                        $century = ($cardinals{$first} * 100) . '-' . (($cardinals{$first} * 100) + 99);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $century);
#                        $decade = (($cardinals{$first} * 100) + $cardinals{$second}) . '-' . ((($cardinals{$first} * 100) + $cardinals{$second}) + 9);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $decade);
#                        $year = (($cardinals{$first} * 100) + $cardinals{$second} + $cardinals{$third}) . '-' . (($cardinals{$first} * 100) + $cardinals{$second} + $cardinals{$third});
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $year);
#                    } elsif($f6XXdata =~ /^($cardinals_keys)( |-o-|-)($cardinals_keys)(?: $|$|, A.D)/i) {
#                        $hash_tag = 'subjects_chronological';
#                        $first = lc($1);
#                        $second = lc($2);
#                        $third = lc($3);
#                        if($second =~ /-o-/) {
#                            $second = '00';
#                        } else {
#                            $second = substr($third, 0, 1) . '0';
#                        }
#                        $century = ($cardinals{$first} * 100) . '-' . (($cardinals{$first} * 100) + 99);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $century);
#                        $decade = (($cardinals{$first} * 100) + $cardinals{$second}) . '-' . ((($cardinals{$first} * 100) + $cardinals{$second}) + 9);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $decade);
#                        $year = (($cardinals{$first} * 100) + $cardinals{$second} + $cardinals{$third}) . '-' . (($cardinals{$first} * 100) + $cardinals{$second} + $cardinals{$third});
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $year); 
#                    } elsif($f6XXdata =~ /^($cardinals_keys)(?: |-)($places_keys)(?: $|$|, A.D)/i) {
#                        $hash_tag = 'subjects_chronological';
#                        $first = lc($1);
#                        $second = lc($2);
#                        $century = ($cardinals{$first} * $places{$second}) . '-' . (($cardinals{$first} * $places{$second}) + 99);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $century);
#                        $decade = ($cardinals{$first} * $places{$second}) . '-' . (($cardinals{$first} * $places{$second}) + 9);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $decade);
#                        $year = ($cardinals{$first} * $places{$second}) . '-' . ($cardinals{$first} * $places{$second});
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $year);
#
#                    } elsif($f6XXdata =~ /^($cardinals_keys)(?: |-)($decades_keys)/i) {
#                        $hash_tag = 'subjects_chronological';
#                        $first = lc($1);
#                        $second = lc($2);
#                        $century = ($cardinals{$first} * 100) . '-' . (($cardinals{$first} * 100) + 99);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $century);
#                        $decade = (($cardinals{$first} * 100) + $decades{$second}) . '-' . ((($cardinals{$first} * 100) + $decades{$second}) + 9);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $decade);
#
#
#                    } elsif($f6XXdata =~ /^($cardinals_keys)(?: |-)($ordinals_keys) century/i) {
#                        $hash_tag = 'subjects_chronological';
#                        $first = lc($1);
#                        $second = lc($2);
#                        $century = ((($cardinals{$first} - 1) + $ordinals{$second}) * 100) . '-' . (((($cardinals{$first} - 1 ) + $ordinals{$second}) * 100) + 99);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $century);
#
#                    } elsif($f6XXdata =~ /^($ordinals_keys) century/i) {
#                        $hash_tag = 'subjects_chronological';
#                        $first = lc($1);
#                        $century = (($ordinals{$first} - 1) * 100) . '-' . ((($ordinals{$first} - 1) * 100) + 99);
#                        uniqueness('coverage_dates_facet', $thesaurus . '\\t', $century);
#                    }

                }
            } elsif($f6XXcode =~ /t/) {
                $this_hash .= ' ' . $f6XXdata;
                $this_subjects_worktitles .= '\t' . $f6XXdata;
                $this_subjects_titles .= '\t' . $f6XXdata;
            } elsif($f6XXcode =~ /[np]/) {
                $this_hash .= ' ' . $f6XXdata;
                $this_subjects_worktitles .= ' ' . $f6XXdata;
                $this_subjects_titles .= ' ' . $f6XXdata;
            } elsif($f6XXcode =~ /[vxy]/) {
                $this_hash .= '--' . $f6XXdata;
                if($f6XXcode =~ /[v]/) {
                    uniqueness_diacritics('genres_facet', $thesaurus, $f6XXdata); 
                } elsif($f6XXcode =~ /[x]/) {
                    uniqueness_diacritics('topics_facet', $thesaurus, $f6XXdata);
                } elsif($f6XXcode =~ /[y]/) {
                    dates_expansion('coverage_dates_facet', $thesaurus, $f6XXdata);
                }
            } elsif($f6XXcode =~ /[z]/) {
                $this_hash .= '--' . $f6XXdata;
                $f6XXdata =~ s/\.$//;
                push(@this_subfield_Zs, $f6XXdata);
            } else {
                $this_hash .= ' ' . $f6XXdata;
            }
        } elsif($f6XXcode =~ /$subparts/) {
            $this_hash .= ' ' . $f6XXdata;
        } elsif($f6XXcode =~ /$relators/) {
            if($f6XXcode =~ /4/) {
                if(exists($relators_hash{$f6XXdata})) {
                    $this_hash .= ' ' . $relators_hash{$f6XXdata};
                } else {
                    $this_hash .= ' ' . $f6XXdata;
                }
            } else {
                $this_hash .= ' ' . $f6XXdata;
            }
        } elsif($f6XXcode =~ /[2]/) {
                #THIS HAS BEEN TAKEN CARE OF EARLIER UNDER SECOND INDICATOR EQUALS 7
        } elsif($f6XXcode =~ /[b-djqu]/) {
            $this_hash .= $f6XXdata;
            $this_subjects_worktitles .= ' ' . $f6XXdata;
        } else {
            $this_hash .= ' ' . $f6XXdata;
        }
    }

    if($#this_subfield_Zs == 2) {
        uniqueness_diacritics('geographics_facet', $thesaurus, $this_subfield_Zs[1] . ' (' . $this_subfield_Zs[2] . ' : ' . $this_subfield_Zs[0] . ') ');
    } elsif($#this_subfield_Zs == 1) {
        uniqueness_diacritics('geographics_facet', $thesaurus, $this_subfield_Zs[1] . ' (' . $this_subfield_Zs[0] . ')');
    } elsif($#this_subfield_Zs == 0) {
        uniqueness_diacritics('geographics_facet', $thesaurus, $this_subfield_Zs[0]);
    } elsif($#this_subfield_Zs > 2 ) {
        $too_many_geographics = '';
        foreach $t ($this_subfield_Zs) {
            $too_many_geographics .= ' ' . $t;
        }
        uniqueness_diacritics('geographics_facet', $thesaurus, $too_many_geographics);
    }
    undef(@this_subfield_Zs);

    if($f6XX->subfield('t') ne '') {
        $this_subjects_worktitles =~ s/\.[ ]*\\t/\\t/g;
        uniqueness_diacritics('subjects_worktitles', '', $this_subjects_worktitles);
        uniqueness_diacritics('subjects_titles', '', $this_subjects_titles);
    }

    uniqueness_diacritics($hash_tag, $thesaurus, $this_hash);

}

foreach $f630 ($record->field('630|699')) {
    $indicator1 = $f630->indicator(1);
    $indicator2 = $f630->indicator(2);
    $this_tag = $f630->tag();
    $this_title = '';
    $this_worktitle = '';

        if(!$f630->subfield('2')) {
            $thesaurus = $thesauri_hash{$indicator2};
        } else {
            $thesaurus = uc($f630->subfield('2'));
        }
    
    foreach $f630_subfields ($f630->subfields()) {
    ($f630code, $f630data ) = @$f630_subfields;
        if($f630code =~ /[adfgk-pr-tvx-z]/) {
            if($f630code =~ /[adnpt]/) {
                $this_worktitle .= $f630data;
            }

            if($f630code =~ /a/) {
                $this_title .= $f630data;
            } elsif($f630code =~ /m/) {
                $this_title .= ' ' . $f630data;
                uniqueness_diacritics('instrumentations_facet', 'UNFORMATTED', $f630data);
            } elsif($f630code =~ /[vx-z]/) {
                $this_title .= '--' . $f630data;
            } else {
                $this_title .= ' ' . $f630data;
            }
        } elsif($f630code =~ /[e4]/) {
            uniqueness_diacritics('roles', '', $f630data . ' (subject)');
        } elsif($f630code =~ /[h9]/) {
            if($this_tag =~ /699/) {
                $this_title .= ' ' . $f630data;
            }
        } elsif($f630code =~ /[2]/) {
            #THIS HAS BEEN TAKEN CARE OF EARLIER UNDER SECOND INDICATOR EQUALS 7
        }
    }

    uniqueness_diacritics('subjects_worktitles', ' ', $this_worktitle);
    uniqueness_diacritics('subjects_titles', ' ', $this_title);
    undef($thesaurus);
}

foreach $f7XX ($record->field('700|71[01]|720|75[34]|79[0-26-8]')) {
    $indicator1 = $f7XX->indicator(1);
    $indicator2 = $f7XX->indicator(2);
    $this_tag = $f7XX->tag;
    $this_hash = '';
    $this_title = '';
    $roles = '';

    if($this_tag =~ /700/) {
        $hash_tag = 'authors_personal';
        $codes = '[a-dfgj-ux5]';
        $relators = '[e4]';
    } elsif($this_tag =~ /710/) {
        $hash_tag = 'authors_corporate';
        $codes = '[a-dfgk-pr-ux5]';
        $relators = '[e4]';
    } elsif($this_tag =~ /711/) {
        $hash_tag = 'authors_conferences';
        $codes = '[ac-gklnps-ux5]';
        $relators = '[4]';
    } elsif($this_tag =~ /720/) {
        $hash_tag = 'authors_uncontrolled';
        $codes = '[a]';
        $relators = '[4]';
    } elsif($this_tag =~ /753/) {
        $hash_tag = 'authors_corporate';
        $codes = '[a-c]';
        $relators = '[ ]';
    } elsif($this_tag =~ /754/) {
        $hash_tag = 'subjects_topical';
        $codes = '[acdxz]';
        $relators = '[ ]';
    } elsif($this_tag =~ /79[012]/) {
        $hash_tag = 'donors';
        $codes = '[a-dfgj-ux5]';
        $relators = '[e4]';
    } elsif($this_tag =~ /796/) {
        $hash_tag = 'authors_personal';
        $codes = '[a-dfgj-ux59]';
        $relators = '[e4]';
    } elsif($this_tag =~ /797/) {
        $hash_tag = 'authors_corporate';
        $codes = '[a-df-hk-pr-ux59]';
        $relators = '[e4]';
    } elsif($this_tag =~ /798/) {
        $hash_tag = 'authors_conferences';
        $codes = '[ac-hklnps-ux59]';
        $relators = '[4]';
    }

    foreach $f7XX_subfields ($f7XX->subfields()) {
    ($f7XXcode, $f7XXdata ) = @$f7XX_subfields;
        if($f7XXcode =~ /$codes/) {
            if($f7XXcode =~ /[a]/) {
                $this_hash .= $f7XXdata;
            } elsif($f7XXcode =~ /l/) {
                $f7XXdata =~ s/(^ |[\.,:;\/]$| $)//g;
                uniqueness_diacritics('languages', ' ', $f7XXdata);
            } elsif($f7XXcode =~ /[m]/) {
                uniqueness_diacritics('instrumentations_facet', 'UNFORMATTED', $f7XXdata);
            } elsif(($f7XXcode =~ /[n]/) && ($this_tag =~ /700|79[06]/)) {
                $this_title .= ' ' . $f7XXdata;
            } elsif($f7XXcode =~ /[kpsw]/) {
                $this_title .= ' ' . $f7XXdata;
            } elsif($f7XXcode =~ /[t]/) {
                $this_title .= $f7XXdata;
            } elsif($f7XXcode =~ /[x]/) {
                if($f7XXdata =~ /([0-9xX]{4}-[0-9xX]{4}).*/) {
                    $this_isn = $1;
                } else {
                    $this_isn = $f7XXdata;
                }
                uniqueness('isn', 'ISSN: \\t', $this_isn);
            } elsif($f7XXcode =~ /[7]/) {
                #'Control subfield'
            } else {
                $this_hash .= ' ' . $f7XXdata;
            }
        } elsif($f7XXcode =~ /$relators/) {
            $this_hash =~ s/([\.,]$)//g;
            if($f7XXcode =~ /4/) {
                if(exists($relators_hash{$f7XXdata})) {
                    uniqueness_diacritics('roles', $this_hash, $relators_hash{$f7XXdata});
                    $roles = $relators_hash{$f7XXdata};
                } else {
                    uniqueness_diacritics('roles', $this_hash, $f7XXdata);
                    $roles = $f7XXdata;
                }
            } else {
                uniqueness_diacritics('roles', $this_hash, $f7XXdata);
                $roles = $f7XXdata;
            }

        } else {
            $this_hash .= $f7XXdata;
        }
    }

    if($this_title ne '') {
        if($indicator2 =~ /2/) {
            $this_title_type = 'titles_analytic';
        } else {
            $this_title_type = 'titles_alternate';
        }
        
        uniqueness_diacritics($this_title_type, '', $this_title);
    }

    if($this_hash ne '') {
        $this_hash =~ s/[\s]*[\.,]*[\s]*$//g;

        if($hash_tag eq 'donors') {
            uniqueness_diacritics($hash_tag, $this_hash, $roles);
        } else {
            uniqueness_diacritics($hash_tag, $this_hash, '');
        }

        if($this_title ne '') {
            $this_hash =~ s/\.[ ]*\\t/\\t/g;
            uniqueness_diacritics('tracings_added_worktitles', $this_hash, $this_title);
        } else {
            if($hash_tag ne 'donors') {
                uniqueness_diacritics('tracings_added_authors', $this_hash, $roles);
            }
        }

    }
}

foreach $f730 ($record->field('730')) {
    $this_tag = $f730->tag();
    $this_work_title = '';
    $uniformtitle_extras = '';

    foreach $f730_subfields ($f730->subfields()) {
    ($f730code, $f730data ) = @$f730_subfields;
            if($f730code =~ /[adgnp]/) {
                $this_work_title .= $f730data;
            } elsif($f730code =~ /f/) {
                #DATE OF WORK -- NOT NEEDED FOR worktitles
                $uniformtitle_extras .= ' ' . $f730data;
            } elsif($f730code =~ /l/) {
                $f730data =~ s/(^ |[\.,:;\/]$| $)//g;
                uniqueness_diacritics('languages', ' ', $f730data);
                $uniformtitle_extras .= ' ' . $f730data;
            } elsif($f730code =~ /k/) {
                $f730data =~ s/(^ |[\.,:;\/]$| $)//g;
                uniqueness_diacritics('genres_facet', 'GENERIC\\t', $f730data);
                $uniformtitle_extras .= ' ' . $f730data;
            } elsif($f730code =~ /m/) {
                $this_work_title .= ' ' . $f730data;
                uniqueness_diacritics('instrumentations_facet', 'UNFORMATTED', $f730data);
                $uniformtitle_extras .= ' ' . $f730data;
            } elsif($f730code =~ /[ors]/) {
                #ARRANGED STATEMENT -- NOT NEEDED FOR worktitles
                #KEY FOR MUSIC -- NOT NEEDED FOR worktitles
                #VERSION -- NOT NEEDED FOR worktitles
                $uniformtitle_extras .= ' ' . $f730data;
            }
    }

    $this_work_title =~ s/^ |([\.,:;\/]|[\.,:;\/] | )$//g;
    $this_work_title =~ s/\.[ ]*\\t/\\t/g;
    uniqueness_diacritics('tracings_added_worktitles', ' ', $this_work_title);
    uniqueness_diacritics('titles_uniform', $this_work_title, $uniformtitle_extras);
}

foreach $f740 ($record->field('740|79[39]')) {
    $indicator1 = $f740->indicator(1);
    $indicator2 = $f740->indicator(2);
    $this_tag = $f740->tag;
    $this_title = '';

    if($this_tag =~ /740/) {
        $codes = '[anp5]';
    } elsif($this_tag =~ /793/) {
        $codes = '[adfgk-pr-tx5]';
    } elsif($this_tag =~ /799/) {
        $codes = '[adf-hk-pr-tx59]';
    }

    foreach $f740_subfields ($f740->subfields()) {
        ($f740code, $f740data ) = @$f740_subfields;
            if($f740code =~ /a/) {
                $this_title .= $f740data;
            } elsif($f740code =~ /l/) {
                $this_title .= ' ' . $f740data;
                $f740data =~ s/(^ |[\.,:;\/]$| $)//g;
                uniqueness_diacritics('languages', ' ',  $f740data);
            } elsif($f740code =~ /m/) {
                $this_title .= ' ' . $f740data;
                uniqueness_diacritics('instrumentations_facet', 'UNFORMATTED', $f740data);
            } else {
                $this_title .= ' ' . $f740data;
            }
    }

    if($indicator2 =~ /2/) {
        $this_title_type = 'titles_analytic';
    } else {
        if($this_tag =~ /79[39]/) {
            $this_title_type = 'titles_uniform';
        } else {
            $this_title_type = 'titles_alternate';
        }
    }

    if($this_tag =~ /793/) {
        uniqueness_diacritics('collections', '', $this_title);
    } else {
        uniqueness_diacritics("tracings_" . $this_title_type, ' ', $this_title);
    }

    uniqueness_diacritics($this_title_type, '', $this_title);
    undef($this_title_type);
}

foreach $f752 ($record->field('752')) {
    $indicator1 = $f752->indicator(1);
    $indicator2 = $f752->indicator(2);
    $this_tag = $f752->tag;
    $this_hash = '';

    foreach $f752_subfields ($f752->subfields()) {
    ($f752code, $f752data ) = @$f752_subfields;
        if($f752code =~ /[a-dfgh]/) {
            if($f752code =~ /[a]/) {
                $subfielda_separator = $f752->subfield('a');
                uniqueness_diacritics('geographics_facet', 'LCSH', $f752data);
            } elsif($f752code =~ /[b]/) {
                $subfieldb_separator = $f752->subfield('b') . ' : ';
                uniqueness_diacritics('geographics_facet', 'LCSH', $f752data . ' (' . $subfielda_separator . ')');
            } elsif($f752code =~ /[c]/) {
                $subfieldc_separator = $f752->subfield('c') . ' : ';
                uniqueness_diacritics('geographics_facet', 'LCSH', $f752data . ' (' . $subfieldb_separator . $subfielda_separator . ')');
            } elsif($f752code =~ /[d]/) {
                $subfieldd_separator = $f752->subfield('d') . ' : ';
                uniqueness_diacritics('geographics_facet', 'LCSH', $f752data . ' (' . $subfieldc_separator . $subfieldb_separator . $subfielda_separator . ')');
            } elsif($f752code =~ /f-h/) {
                $count++;
                if($count > 0) {
                    $separator = ', ';
                }
                $geographic_parentheses = ' (' . $subfieldd_separator . $subfieldc_separator . $subfieldb_separator . $subfielda_separator . ')';

                foreach $subfieldg ($f752->subfield($f752code)) {
                    uniqueness_diacritics('geographics_facet', '', $subfieldf . $geographic_parentheses);
                    $geographic_parentheses = $subfieldf . $separator . $geographic_parentheses;
                }
                undef($count);
                undef($geographic_parentheses);
            }

        } else {
                uniqueness_diacritics('geographics_facet', 'LCSH', $f752data);
        }
    }
                undef($subfielda_separator);
                undef($subfieldb_separator);
                undef($subfieldc_separator);
                undef($subfieldd_separator);
}

foreach $f760 ($record->field('76[02]')) {
    $indicator1 = $f760->indicator(1);
    $indicator2 = $f760->indicator(2);
    $this_tag = $f760->tag;
    $this_label = '';
    $this_series_hash = '';
    $this_series_title = '';
    $this_series_extra = '';

    if($f760->tag =~ /760/) {
        $this_label = 'Subseries of:';
    } else {
        $this_label = 'Has subseries:';
    }

    foreach $f760_subfields ($f760->subfields()) {
    ($f760code, $f760data ) = @$f760_subfields;
            if($f760code =~ /[a]/) {
                $this_series_hash .= $f760data;
            } elsif($f760code =~ /[cknpsw]/) {
                $this_series_title .= ' ' . $f760data;
            } elsif($f760code =~ /[t]/) {
                $this_series_title .= $f760data;
            } elsif($f760code =~ /[i]/) {
                $this_label = $f760data;
            } elsif($f760code =~ /[w]/) {
                $this_series_extra .= ' ' . $f760data;
            } elsif($f760code =~ /[x]/) {
                $this_series_extra .= ' ' . $f760data;
                if($f760data =~ /([0-9xX]{4}-[0-9xX]{4}).*/) {
                    $this_isn = $1;
                } else {
                    $this_isn = $f760data;
                }
                uniqueness('isn', 'ISSN: \\t', $this_isn);
            } else {
                $this_series_extra .= ' ' . $f760data;
            }
    }

        uniqueness_diacritics('notes_public', "$this_label ", $this_series_hash . $this_series_title . $this_series_extra);
    $this_series_hash =~ s/[\s]*[\.,]*[\s]*$//g;
    $this_series_title =~ s/([\s]*[\.,:;\/]*[\s]*|[\s]*[\.,:;\/]*[\s]* |[\s]*)$//g;

    uniqueness_diacritics('series_traced', $this_series_hash . '\\t' . $this_series_title, $this_series_volume);

        uniqueness_diacritics('titles_series', '', $this_series_title);

}

foreach $f765 ($record->field('76[57]')) {
    $indicator1 = $f765->indicator(1);
    $indicator2 = $f765->indicator(2);
    $this_tag = $f765->tag;
    $this_label = '';
    $this_hash = '';
    $this_title = '';
    $this_extra = '';

    if($f765->tag =~ /765/) {
        $this_label = 'Translation of:';
    } else {
        $this_label = 'Translated as:';
    }

    foreach $f765_subfields ($f765->subfields()) {
    ($f765code, $f765data ) = @$f765_subfields;
            if($f765code =~ /[a]/) {
                $this_hash .= $f765data;
            } elsif($f765code =~ /[cknpsw]/) {
                $this_title .= ' ' . $f765data;
            } elsif($f765code =~ /[t]/) {
                $this_title .= $f765data;
            } elsif($f765code =~ /[i]/) {
                $this_label = $f765data;
            } elsif($f765code =~ /[w]/) {
                $this_extra .= ' ' . $f765data;
            } elsif($f765code =~ /[x]/) {
                $this_extra .= ' ' . $f765data;
                if($f765data =~ /([0-9xX]{4}-[0-9xX]{4}).*/) {
                    $this_isn = $1;
                } else {
                    $this_isn = $f765data;
                }
                uniqueness('isn', 'ISSN: \\t', $this_isn);
            } else {
                $this_extra .= ' ' . $f765data;
            }
    }

        uniqueness_diacritics('notes_public', "$this_label ", $this_hash . " " . $this_title . " " . $this_extra);

    if($this_hash ne '') {
        uniqueness_diacritics('authors_uncontrolled', '', $this_hash);
    }

    if($this_title ne '') {
        uniqueness_diacritics('titles_alternate', '', $this_title);
    }
}

foreach $f77X ($record->field('77[02-7]|78[05-7]')) {
    $indicator1 = $f77X->indicator(1);
    $indicator2 = $f77X->indicator(2);
    $this_tag = $f77X->tag;
    $this_label = '';
    $this_hash = '';
    $this_title = '';
    $this_extra = '';
    
    if($f77X->tag =~ /770/) {
        $this_label = 'Has supplement:';
    } elsif($f77X->tag =~ /772/) {
        $this_label = 'Supplement to:';
    } elsif($f77X->tag =~ /773/) {
        $this_label = 'In:';
    } elsif($f77X->tag =~ /774/) {
        $this_label = 'Constituent unit:';
    } elsif($f77X->tag =~ /775/) {
        $this_label = 'Other edition available:';
    } elsif($f77X->tag =~ /776/) {
        $this_label = 'Issued in other form:';
    } elsif($f77X->tag =~ /777/) {
        $this_label = 'Issued with:';
    } elsif($f77X->tag =~ /786/) {
        $this_label = 'Data source:';
    } elsif($f77X->tag =~ /787/) {
        $this_label = 'Related item:';
    }

    if($this_tag =~ /77[0246]/) {
        $codes = '[abdghikm-or-uw-z7]';
    } elsif($this_tag =~ /773/) {
        $codes = '[abdghikm-uw-z37]';
    } elsif($this_tag =~ /775/) {
        $codes = '[abd-ikm-or-uw-z7]';
    } elsif($this_tag =~ /777/) {
        $codes = '[abdghikm-ostw-y7]';
    } elsif($this_tag =~ /786/) {
        $codes = '[abdgh-km-pr-z7]';
    } elsif($this_tag =~ /787/) {
        $codes = '[abdghim-or-uw-z7]';
    }

    foreach $f77X_subfields ($f77X->subfields()) {
        ($f77Xcode, $f77Xdata ) = @$f77X_subfields;
            if($f77Xcode =~ /[a]/) {
                $this_hash .= $f77Xdata;
            } elsif($f77Xcode =~ /[cknpsw]/) {
                $this_title .= ' ' . $f77Xdata;
            } elsif($f77Xcode =~ /[t]/) {
                $this_title .= $f77Xdata;
            } elsif($f77Xcode =~ /[i]/) {
                $this_label = $f77Xdata;
            } elsif($f77Xcode =~ /[w]/) {
                $this_extra .= ' ' . $f77Xdata;
            } elsif($f77Xcode =~ /[x]/) {
                $this_extra .= ' ' . $f77Xdata;
                if($f77Xdata =~ /([0-9xX]{4}-[0-9xX]{4}).*/) {
                    $this_isn = $1;
                } else {
                    $this_isn = $f77Xdata;
                }
                uniqueness('isn', 'ISSN: \\t', $this_isn);
            } elsif($f77Xcode =~ /[7]/) {
                #'Control subfield'
            } else {
                $this_extra .= ' ' . $f77Xdata;
            }
    }

    uniqueness_diacritics('tracings_titles_alternate', "$this_label ", $this_hash . " " . $this_title . " " . $this_extra);

    if($this_hash ne '') {
        uniqueness_diacritics('authors_uncontrolled', '', $this_hash);
    }

    if($this_title ne '') {
        uniqueness_diacritics('titles_alternate', '', $this_title);
    }

}

foreach $f780 ($record->field('78[05]')) {
    $indicator1 = $f780->indicator(1);
    $indicator2 = $f780->indicator(2);
    $this_tag = $f780->tag;
    $this_label = '';
    $this_hash = '';
    $this_title = '';
    $this_extra = '';

    if($f780->tag =~ /780/) {
        if($indicator2 =~ /0/) {
            $this_label = 'Continues:';
        } elsif($indicator2 =~ /1/) {
            $this_label = 'Continues in part:';
        } elsif($indicator2 =~ /2/) {
            $this_label = 'Supersedes:';
        } elsif($indicator2 =~ /3/) {
            $this_label = 'Supersedes in part:';
        } elsif($indicator2 =~ /4/) {
            $field_780_indicator2_4++;
            if($field_780_indicator2_4 <= 1) {
            $this_label = 'Formed by the union of:';
            } else {
                $this_label = 'and:';
            }
        } elsif($indicator2 =~ /5/) {
            $this_label = 'Absorbed:';
        } elsif($indicator2 =~ /6/) {
            $this_label = 'Absorbed in part:';
        } elsif($indicator2 =~ /7/) {
            $this_label = 'Separated from:';
        }
    } elsif($f780->tag =~ /785/) {
        if($indicator2 =~ /0/) {
            $this_label = 'Continued by:';
        } elsif($indicator2 =~ /1/) {
            $this_label = 'Continued in part by:';
        } elsif($indicator2 =~ /2/) {
            $this_label = 'Superseded by:';
        } elsif($indicator2 =~ /3/) {
            $this_label = 'Superseded in part by:';
        } elsif($indicator2 =~ /4/) {
            $this_label = 'Absorbed by:';
        } elsif($indicator2 =~ /5/) {
            $this_label = 'Absorbed in part by:';
        } elsif($indicator2 =~ /6/) {
            $field_785_indicator2_6++;
            if($field_785_indicator2_6 <= 1) {
                $this_label = 'Split into:';
            } else {
                $this_label = 'and:';
            }
        } elsif($indicator2 =~ /7/) {
            $field_785_indicator2_7++;
            if($field_785_indicator2_6 <= 1) {
                $this_label = 'Merged with:';
            } else {
                $this_label = 'to form:';
            }
        } elsif($indicator2 =~ /8/) {
            $this_label = 'Changed back to:';
        }
    }

    foreach $f780_subfields ($f780->subfields()) {
        ($f780code, $f780data ) = @$f780_subfields;
            if($f780code =~ /[a]/) {
                $this_hash .= $f780data;
            } elsif($f780code =~ /[cknpsw]/) {
                $this_title .= ' ' . $f780data;
            } elsif($f780code =~ /[t]/) {
                $this_title .= $f780data;
            } elsif($f780code =~ /[i]/) {
                $this_label = $f780data;
            } elsif($f780code =~ /[g]/) {
                if(($this_label eq 'Continued by:') && ($f780data =~ /[0-9]{4}/)) {
                    $this_label = 'Continued in ' . $f780data . 'by:';
                } else {
                    $this_extra .= ' ' . $f780data;
                }
            } elsif($f780code =~ /[w]/) {
                $this_extra .= ' ' . $f780data;
            } elsif($f780code =~ /[x]/) {
                $this_extra .= ' ' . $f780data;
                if($f780data =~ /([0-9xX]{4}-[0-9xX]{4}).*/) {
                    $this_isn = $1;
                } else {
                    $this_isn = $f780data;
                }
                uniqueness('isn', 'ISSN: \\t', $this_isn);
            } elsif($f780code =~ /[7]/) {
                #'Control subfield'
            } else {
                $this_extra .= ' ' . $f780data;
            }
    }


    if($this_hash ne '') {
        uniqueness_diacritics('authors_uncontrolled', '', $this_hash);
    }

    if($this_title ne '') {
        uniqueness_diacritics('titles_alternate', '', $this_title);
    }

    if($this_tag =~ /780/) {
        uniqueness_diacritics('tracings_titles_earlier', "$this_label ", $this_hash . " " . $this_title . " " . $this_extra);
    } elsif($this_tag =~ /785/) {
        uniqueness_diacritics('tracings_titles_later', "$this_label ", $this_hash . " " . $this_title . " " . $this_extra);
    }

}

foreach $f8XX ($record->field('800|810|811|89[6-8]')) {
    $this_tag = $f8XX->tag();
    $this_series_title = '';
    $this_series_volume = '';
    $this_series_hash = '';
    $f8XX_score++;

    if($this_tag =~ /800|896/) {
        $hash_tag = 'authors_personal';
        $codes = '[a-djqu]';
        $title_parts = '[f-hk-pr-t]';
        $relators = '[e4]';
    } elsif($this_tag =~ /810|897/) {
        $hash_tag = 'authors_corporate';
        $codes = '[a-du]';
        $title_parts = '[fgk-pr-t]';
        $relators = '[e4]';
    } elsif($this_tag =~ /811|898/) {
        $hash_tag = 'authors_conferences';
        $codes = '[ac-equ]';
        $title_parts = '[f-hklnpst]';
        $relators = '[4]';
    }

    foreach $f8XX_subfields ($f8XX->subfields()) {
    ($f8XXcode, $f8XXdata ) = @$f8XX_subfields;
        if($f8XXcode =~ /$codes/) {
            if($f8XXcode =~ /a/) {
                $this_series_hash .= $f8XXdata;
            } else {
                $this_series_hash .= ' ' . $f8XXdata;
            }
        } elsif($f8XXcode =~ /$relators/) {
            $this_series_hash =~ s/([\.,]$)//g;
            if($f8XXcode =~ /4/) {
                if(exists($relators_hash{$f8XXdata})) {
                    uniqueness_diacritics('roles', $this_series_hash, $relators_hash{$f8XXdata});
                } else {
                    uniqueness_diacritics('roles', $this_series_hash, $f8XXdata);
                }
            } else {
                uniqueness_diacritics('roles', $this_series_hash, $f8XXdata);
            }
        } elsif($f8XXcode =~ /$title_parts/) {
            if($f8XXcode =~ /l/) {
                $f8XXdata =~ s/(^ |[\.,:;\/]$| $)//g;
                uniqueness_diacritics('languages', ' ', $f8XXdata);
            } elsif($f8XXcode =~ /m/) {
                $this_series_title .= $f8XXdata . ' ';
                uniqueness_diacritics('instrumentations_facet', 'UNFORMATTED', $f8XXdata);
            } else {
                $this_series_title .= $f8XXdata . ' ';
            }
        } elsif($f8XXcode =~ /[v]/) {
                $this_series_volume .= $f8XXdata . ' ';
        } else {
            $this_series_title .= $f8XXdata . ' ';
        }
    }

    $this_series_hash =~ s/[\s]*[\.,]*[\s]*$//g;
    $this_series_title =~ s/([\s]*[\.,:;\/]*[\s]*|[\s]*[\.,:;\/]*[\s]* |[\s]*)$//g;

    uniqueness_diacritics('series_traced', $this_series_hash . '\\t' . $this_series_title, $this_series_volume);
    uniqueness_diacritics($hash_tag, $this_series_hash, '');
    uniqueness_diacritics('titles_series', '', $this_series_title);
}

foreach $f830 ($record->field('830|899')) {
    $indicator1 = $f830->indicator(1);
    $indicator2 = $f830->indicator(2);
    $this_tag = $f830->tag();
    $this_series_title = '';
    $this_series_volume = '';

    foreach $f830_subfields ($f830->subfields()) {
    ($f830code, $f830data ) = @$f830_subfields;
        if($f830code =~ /[adfgkn-pt]/) {
            if($f830code =~ /a/) {
                $this_series_title .= $f830data;
            } else {
                $this_series_title .= ' ' . $f830data;
            }
        } elsif($f830code =~ /l/) {
            $f830data =~ s/(^ |[\.,:;\/]$| $)//g;
            uniqueness_diacritics('languages', ' ', $f830data);
        } elsif($f830code =~ /m/) {
            $this_series_title .= ' ' . $f830data;
            uniqueness_diacritics('instrumentations_facet', 'UNFORMATTED', $f830data);
        } elsif($f830code =~ /[v]/) {
                $this_series_volume .= $f830data . ' ';
        }
    }

    $this_series_title =~ s/([\s]*[\.,:;\/]*[\s]*|[\s]*[\.,:;\/]*[\s]* |[\s]*)$//g;
    uniqueness_diacritics('series_traced', '\\t' . $this_series_title, $this_series_volume);
    uniqueness_diacritics('titles_series', '', $this_series_title);

}

foreach $f856 ($record->field('856|956')) {
    $indicator1 = $f856->indicator(1);
    $indicator2 = $f856->indicator(2);
    $this_tag = $f856->tag;
    $this_hash = '';
    $this_label = 'Connect to resource online. ';

#    if($indicator1 =~ /0/) {
#        $this_label = 'Access method: E-mail. ';
#    } elsif($indicator1 =~ /1/) {
#        $this_label = 'Access method: FTP. ';
#    } elsif($indicator1 =~ /2/) {
#        $this_label = 'Access method: Remote login. ';
#    } elsif($indicator1 =~ /3/) {
#        $this_label = 'Access method: Dial-up. ';
#    } elsif($indicator1 =~ /4/) {
#        $this_label = 'Access method: HTTP. ';
#    } elsif(($indicator1 =~ /7/) && ($f856->subfield('2'))){
#        $this_label = 'Access method: ' . $f856->subfield('2') . '. ';
#    }

    if($indicator2 =~ /1/) {
        $this_label .= 'Version of resource. ';
    } elsif($indicator2 =~ /2/) {
        $this_label .= 'Related resource. ';
    }

    foreach $f856_subfields ($f856->subfields()) {
    ($f856code, $f856data ) = @$f856_subfields;
            if($f856code =~ /[a-dfh-tv-z3]/) {
                $this_label .= $f856data;
            } elsif($f856code =~ /[u]/) {
                $this_hash .= $f856data;
            }
    }

    uniqueness_diacritics('urls', "$this_hash ", $this_label);
}

foreach $f907 ($record->field('907')) {
    foreach $f907_subfields ($f907->subfields()) {
    ($f907code, $f907data ) = @$f907_subfields;
            if($f907code =~ /a/) {
                $query_insert{'local_control_number'} = $institution . substr($f907data, 1, 8);
            } elsif($f907code =~ /b/) {
                $query_insert{'updated_date'} = $f907data;
            } elsif($f907code =~ /c/) {
                $query_insert{'created_date'} = $f907data;
            }
    }

}

foreach $f998 ($record->field('998')) {
    foreach $f998_subfields ($f998->subfields()) {
    ($f998code, $f998data ) = @$f998_subfields;
            if($f998code =~ /a/) {
                uniqueness('bloc', '', $institution . '\t' . $f998data);
            } elsif($f998code =~ /b/) {
                $query_insert{'cat_date'} = $f998data;
            } elsif($f998code =~ /e/) {
                $query_insert{'bib_status'} = $f998data;
            }
    }
}

    foreach $date_field( cat_date, updated_date, created_date ) {
        if(substr($query_insert{$date_field}, 6, 1) >= 8) {
        $prepend = '19';
        } else {
        $prepend = '20';
        }
        if($query_insert{$date_field} =~ /([0-9]{2})-([0-9]{2})-([0-9]{2})/) {
            $query_insert{$date_field} = $prepend . $3 . '-' . $1 . '-' . $2;
        }
    }

if($query_insert{'formats'} eq '') {
    $query_insert{'formats'} = 'Print';
}

####START: PRINT VALUES TO BATCH FILES

@keys = keys %query_insert;
@values = values %query_insert;

$mysql_keys = join(", ", @keys);
$mysql_values = join("\t", @values);
$mysql_values =~ s/\\\t/\\ \t/g;

print $batch_loader $mysql_values . "\n";


foreach $raw_tracing_line (@raw_tracings_array) {

print $raw_tracings_loader $query_insert{'local_control_number'} . "\t" . $query_insert{'record_owner'} . "\t" . $query_insert{'owner_level'} . "\t" . $raw_tracing_line . "\n";;

}

####END: PRINT VALUES TO BATCH FILES

                undef($mysql_values);
                undef(%query_insert);
                undef(@keys);
                undef(@values);
                undef(@raw_tracings_array);
                undef($tracing_sequence);
}

}

    close(IN) || "cannot close $next_file: $!";

unlink($next_file);

        $batch_loader->autoflush(1);
        close($batch_loader, 2) || "cannot close $batch_loader: $!";

        $raw_tracings_loader->autoflush(1);
        close($raw_tracings_loader, 2) || "cannot close $raw_tracings_loader: $!";

}

$bibs_keys_textfiles = $baseurl . '/' . $folder_name . '_textfiles/bibs.key';

$bibs_keys_loader = IO::File->new("> $bibs_keys_textfiles") || die "cannot open $bibs_keys_textfiles for writing: $!\n";
binmode( $bibs_keys_loader, ':utf8' );
flock($bibs_keys_loader, 2);

    print $bibs_keys_loader $mysql_keys . "\n";

        $bibs_keys_loader->autoflush(1);
        close($bibs_keys_loader, 2) || "cannot close $bibs_keys_loader: $!";


@current_time = localtime;

# connection for the select
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host", $db_user, $db_pass, { mysql_enable_utf8 => 1 }) or die "couldn't connect to database: " . DBI->errstr;

($HOUR, $MIN, $SEC) = (localtime)[2, 1, 0];
$end_of_file_creation_time = "$HOUR:$MIN:$SEC";

if ($ARGV[0] =~ /CREATING/) {
    $sth1= $dbh->do("TRUNCATE TABLE $db_local_tables_hash{'bibs_record_table'}");
    $sth1= $dbh->do("TRUNCATE TABLE $db_local_tables_hash{'holdings_record_table'}");
    $this_table = $db_local_tables_hash{'bibs_record_table'};
} else {
    $sth1= $dbh->do("TRUNCATE TABLE $db_local_tables_hash{'bibs_record_temp_table'}");
    $this_table = $db_local_tables_hash{'bibs_record_temp_table'};
}

$where = $baseurl . '/' . $folder_name . '_textfiles';

#####START: INITIAL TABLE LOAD

#if ($ARGV[0] =~ /CREATING/) {
#    $dbh->do("ALTER TABLE " . $this_table . " DISABLE KEYS");
#}

while (defined($next_file = <$where/*.out>)) {

    $sth2 = $dbh->do("LOAD DATA INFILE '" . $next_file . "' REPLACE INTO TABLE $this_table FIELDS TERMINATED BY '\t' (" . $mysql_keys . ")") || print "initial load sth2 died in local\n";

unlink($next_file);

}

unlink($bibs_keys_textfiles);

$that_table = $db_local_tables_hash{'universal_ids_table'};

        $querystring = "UPDATE $this_table, $that_table
            SET $this_table.universal_identifier = $that_table.universal_identifier,
                $this_table.oclc = $that_table.true_oclc
           WHERE $this_table.local_control_number = $that_table.local_control_number";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute()
            or die("couldn't execute statement: ". $dbh->errstr);

        $query->finish();

#if ($ARGV[0] =~ /CREATING/) {
#    $dbh->do("ALTER TABLE " . $this_table . " ENABLE KEYS");
#}

$that_table = $db_local_tables_hash{'tocs_970s_table'};

        $querystring = "UPDATE $this_table, $that_table
            SET $this_table.authors_tocs = $that_table.authors_tocs,
                $this_table.titles_tocs = $that_table.titles_tocs,
                $this_table.tocs = $that_table.tocs
            WHERE $this_table.local_control_number = $that_table.local_control_number";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute()
            or die("couldn't execute statement: ". $dbh->errstr);

#####END: INITIAL TABLE LOAD

######START: UPDATE ACTUAL BIB TABLE WITH DATA IN TEMP TABLE

if ($ARGV[0] ne 'CREATING') {

$this_table = $db_local_tables_hash{'bibs_record_table'};
$that_table = $db_local_tables_hash{'bibs_record_temp_table'};

    $querystring = "SHOW FIELDS FROM $this_table";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute()
            or die("couldn't execute statement: ". $dbh->errstr);

    while ( @row = $query->fetchrow_array ) {
        push (@keys, $row[0]); ## field name
    }

$query->finish();

pop(@keys);

$mysql_keys = join(", ", @keys);

@update_strings = ();

    foreach $field_name (@keys) {
        push(@update_strings, $this_table . '.' . $field_name . ' = ' . $that_table . '.' . $field_name);
    }
    
    $temp_updates = join(", ", @update_strings);

        $querystring = "UPDATE $this_table, $that_table
            SET $temp_updates
            WHERE $this_table.local_control_number = $that_table.local_control_number";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute()
            or die("couldn't execute statement: ". $dbh->errstr);

        $query->finish();

        $querystring = "DELETE $that_table.*
            FROM $that_table
            LEFT JOIN $this_table
            ON $that_table.local_control_number = $this_table.local_control_number
            WHERE $that_table.local_control_number = $this_table.local_control_number";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute()
            or print "couldn't execute statement: $querystring";

        $query->finish();

        $querystring = "INSERT $this_table
            ($mysql_keys)
            SELECT $mysql_keys from $that_table";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute()
            or print "couldn't execute statement: $querystring";

        $query->finish();

}

######END: UPDATE ACTUAL BIB TABLE WITH DATA IN TEMP TABLE

#######START: LOADING TRACINGS FILES INTO TRACINGS TABLE

$last_update_time = ($current_time[5] + 1900) . "-" . ($current_time[4] + 1) . "-$current_time[3] $current_time[2]:$current_time[1]:$current_time[0]";

print "mysql_tracings_keys last_update_time: " . $last_update_time . "\n";

$this_table = $db_local_tables_hash{'tracings_record_table'};
$recordcount = 0;

$position = 0;

    $querystring = "SHOW FIELDS FROM $this_table";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute()
            or die("couldn't execute statement: ". $dbh->errstr);

    while ( @row = $query->fetchrow_array ) {
        push (@tracings_keys, $row[0]); ## field name
    }

$query->finish();

pop(@tracings_keys);


    $mysql_tracings_keys = join(", ", @tracings_keys);

print 'mysql_tracings_keys: ' . $mysql_tracings_keys  . "\n";

    $where = $baseurl . '/tracings_textfiles';

#if ($ARGV[0] =~ /CREATING/) {
#    $dbh->do("ALTER TABLE " . $db_local_tables_hash{'tracings_record_table'} . " DISABLE KEYS");
#}

while (defined($next_file = <$where/*.out>)) {

   $sth4 = $dbh->do("LOAD DATA INFILE '" . $next_file . "' REPLACE INTO TABLE $db_local_tables_hash{'tracings_record_table'} FIELDS TERMINATED BY '\t' (" . $mysql_tracings_keys . ")") || print "died again sth4 local\n";

unlink($next_file);

}

#if ($ARGV[0] =~ /CREATING/) {
#    $dbh->do("ALTER TABLE " . $db_local_tables_hash{'tracings_record_table'} . " ENABLE KEYS");
#}

########END: LOADING TRACINGS FILES INTO TRACINGS TABLE

#########START: CREATING FILES FOR LOAD INTO LOCAL HOLDINGS TABLE AND CONSORTIUM BIBS AND HOLDINGS TABLES

if ($consortium ne '') {

($HOUR, $MIN, $SEC) = (localtime)[2, 1, 0];
$start_external_update_time = "$HOUR:$MIN:$SEC";

}

$last_update_time = ($current_time[5] + 1900) . "-" . ($current_time[4] + 1) . "-$current_time[3] $current_time[2]:$current_time[1]:$current_time[0]";

print "last_update_time: " . $last_update_time . "\n";

$this_table = $db_local_tables_hash{'holdings_record_table'};
$recordcount = 0;

$position = 0;

    $querystring = "SHOW FIELDS FROM $this_table";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute()
            or die("couldn't execute statement: ". $dbh->errstr);

    while ( @row = $query->fetchrow_array ) {
        push (@holdings_keys, $row[0]); ## field name
    }

$query->finish();

pop(@holdings_keys);

$this_table = $db_local_tables_hash{'bibs_record_table'};

    $querystring = "SHOW FIELDS FROM $this_table";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute()
            or die("couldn't execute statement: ". $dbh->errstr);

    %bibs2holdings_hash = ();

    while ( @row = $query->fetchrow_array ) {
        push (@fields_array, $row[0]); ## field name
           $bibs2holdings_hash{$row[0]} = $position;
        $position++;
    }

        $query->finish();

pop(@fields_array);

    $fields_array_string = join(", ", @fields_array);
    $mysql_holdings_keys = join(", ", @holdings_keys);

$holdings_keys_textfiles = $baseurl . '/holdings_textfiles/holdings.key';

$holdings_keys_loader = IO::File->new("> $holdings_keys_textfiles") || die "cannot open $holdings_keys_textfiles for writing: $!\n";
binmode( $holdings_keys_loader, ':utf8' );
flock($holdings_keys_loader, 2);

    print $holdings_keys_loader $mysql_holdings_keys . "\n";

        $holdings_keys_loader->autoflush(1);
        close($holdings_keys_loader, 2) || "cannot close $holdings_keys_loader: $!";

if ($consortium ne '') {

$consortium_holdings_keys_textfiles = $baseurl . '/consortium_holdings_textfiles/consortiumholdings.key';
$consortium_bibs_keys_textfiles = $baseurl . '/consortium_bibs_textfiles/consortiumbibs.key';

$consortium_holdings_keys_loader = IO::File->new("> $consortium_holdings_keys_textfiles") || die "cannot open $consortium_holdings_keys_textfiles for writing: $!\n";
binmode( $consortium_holdings_keys_loader, ':utf8' );
flock($consortium_holdings_keys_loader, 2);

$consortium_bibs_keys_loader = IO::File->new("> $consortium_bibs_keys_textfiles") || die "cannot open $consortium_bibs_keys_textfiles for writing: $!\n";
binmode( $consortium_bibs_keys_loader, ':utf8' );
flock($consortium_bibs_keys_loader, 2);

    print $consortium_holdings_keys_loader $mysql_holdings_keys . "\n";
    print $consortium_bibs_keys_loader $fields_array_string . "\n";

        $consortium_holdings_keys_loader->autoflush(1);
        close($consortium_holdings_keys_loader, 2) || "cannot close $consortium_holdings_keys_loader: $!";

        $consortium_bibs_keys_loader->autoflush(1);
        close($consortium_bibs_keys_loader, 2) || "cannot close $consortium_bibs_keys_loader: $!";
}

if ($ARGV[0] eq 'CREATING') {
    $IN_string = "'n', 's', 'z', 'x'";
} else {
    $IN_string = "'n', 's', 'z'";
}

$limit_start = 0;
$rownumber = 1;
while($rownumber > 0) {

        $querystring = "SELECT * from $this_table
            WHERE solr_update >= ? 
            AND bib_status NOT IN ($IN_string)
            LIMIT ?, 10000";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute($last_update_time, $limit_start)
            or die("couldn't execute statement: ". $dbh->errstr);

$limit_start += 10000;
$rownumber = $query->rows();

    print "rownumber: " . $rownumber . "\n";

while(@query_array = $query->fetchrow_array()) {
    $int = $recordcount / 10000;

    if ($int =~ m/^\d+$/) {

        if($recordcount > 0) {
        $holdings_batch_loader->autoflush(1);
        close($holdings_batch_loader, 2) || "cannot close $holdings_batch_loader: $!";
        }

$holdings_batch_textfiles = $baseurl . '/holdings_textfiles/holdings' . $recordcount .  '.txt';

$holdings_batch_loader = IO::File->new("> $holdings_batch_textfiles") || die "cannot open $holdings_batch_textfiles for writing: $!\n";
binmode( $holdings_batch_loader, ':utf8' );
flock($holdings_batch_loader, 2);


if ($consortium ne '') {

        if($recordcount > 0) {

        $consortium_holdings_batch_loader->autoflush(1);
        close($consortium_holdings_batch_loader, 2) || "cannot close $consortium_holdings_batch_loader: $!";

        $consortium_bibs_batch_loader->autoflush(1);
        close($consortium_bibs_batch_loader, 2) || "cannot close $consortium_bibs_batch_loader: $!";
        }

$consortium_holdings_batch_textfiles = $baseurl . '/consortium_holdings_textfiles/consortium_from_1' . $folder_name . '_' . $recordcount .  '.txt';
$consortium_bibs_batch_textfiles = $baseurl . '/consortium_bibs_textfiles/consortium_from_1' . $folder_name . '_' . $recordcount .  '.txt';

$consortium_holdings_batch_loader = IO::File->new("> $consortium_holdings_batch_textfiles") || die "cannot open $consortium_holdings_batch_textfiles for writing: $!\n";
binmode( $consortium_holdings_batch_loader, ':utf8' );
flock($consortium_holdings_batch_loader, 2);

$consortium_bibs_batch_loader = IO::File->new("> $consortium_bibs_batch_textfiles") || die "cannot open $consortium_bibs_batch_textfiles for writing: $!\n";
binmode( $consortium_bibs_batch_loader, ':utf8' );
flock($consortium_bibs_batch_loader, 2);
}

    }

$recordcount++;

pop(@query_array);

foreach(@query_array) {
    s/\t/\\t/g;
}

@blocs = split("\n", $query_array[$bibs2holdings_hash{'bloc'}]);

foreach $this_bloc (@blocs) {

        @holdings_array = ();
    foreach $this_holding_key(@holdings_keys) {

        if($this_holding_key =~ /lib_has|latest_received/) {
            push(@holdings_array, '');
        } elsif($this_holding_key =~ /hloc/) {
            push(@holdings_array, $this_bloc);
        } elsif($this_holding_key =~ /holding_status/) {
            push(@holdings_array, $query_array[$bibs2holdings_hash{'bib_status'}]);
        } else {
            push(@holdings_array, $query_array[$bibs2holdings_hash{$this_holding_key}]);
        }


    }

        $query_array_string = join("\t", @query_array);
        $query_array_string =~ s/\\\t/\\ \t/g;
        $query_array_string =~ s/\n/\\n/g;

        $holdings_array_string = join("\t", @holdings_array);
        $holdings_array_string =~ s/\\\t/\\ \t/g;
        $holdings_array_string =~ s/\n/\\n/g;

        print $holdings_batch_loader $holdings_array_string . "\n";

if ($consortium ne '') {

        print $consortium_holdings_batch_loader $holdings_array_string . "\n";
        print $consortium_bibs_batch_loader $query_array_string . "\n";
}

        $line_count++;
}
    }

}

$query->finish();

if($recordcount > 0) {

        $mysql_holdings_keys = join(", ", @holdings_keys);

        $holdings_batch_loader->autoflush(1);
        close($holdings_batch_loader, 2) || "cannot close $holdings_batch_loader: $!";

if ($consortium ne '') {

        $consortium_holdings_batch_loader->autoflush(1);
        close($consortium_holdings_batch_loader, 2) || "cannot close $consortium_holdings_batch_loader: $!";

        $consortium_bibs_batch_loader->autoflush(1);
        close($consortium_bibs_batch_loader, 2) || "cannot close $consortium_bibs_batch_loader: $!";
}

########END: CREATING FILES FOR LOAD INTO LOCAL HOLDINGS TABLE AND CONSORTIUM BIBS AND HOLDINGS TABLES

#########START: LOADING FILES INTO LOCAL HOLDINGS TABLE

    $where = $baseurl . '/holdings_textfiles';

#if ($ARGV[0] =~ /CREATING/) {
#    $dbh->do("ALTER TABLE " . $db_local_tables_hash{'holdings_record_table'} . " DISABLE KEYS");
#}

while (defined($next_file = <$where/*.txt>)) {

   $sth5 = $dbh->do("LOAD DATA INFILE '" . $next_file . "' REPLACE INTO TABLE $db_local_tables_hash{'holdings_record_table'} FIELDS TERMINATED BY '\t' (" . $mysql_holdings_keys . ")") || print "died again sth5 local\n";

unlink($next_file);

}

unlink($holdings_keys_textfiles);

#if ($ARGV[0] =~ /CREATING/) {
#    $dbh->do("ALTER TABLE " . $db_local_tables_hash{'holdings_record_table'} . " ENABLE KEYS");
#}

##########END: LOADING FILES INTO LOCAL HOLDINGS TABLE


}

if ($consortium ne '') {

($HOUR, $MIN, $SEC) = (localtime)[2, 1, 0];
$end_external_update_time = "$HOUR:$MIN:$SEC";

}

$dbh->disconnect;

($HOUR, $MIN, $SEC) = (localtime)[2, 1, 0];
$end_time = "$HOUR:$MIN:$SEC";

print "Start time: " . $start_time . "\nEnd of file creation time: " . $end_of_file_creation_time . "\nEnd time: " . $end_time . "\n Number of files: " . $total_number_of_files . "\n";
print "start_external_update_time: " . $start_external_update_time . "\n";
print "end_external_update_time: " . $end_external_update_time . "\n";

close (STDERR) || "cannot close $errors_outs: $!";
close (STDOUT) || "cannot close $prints_outs: $!";

#SUBROUTINES

sub hashes_and_arrays {

$hash_files = $baseurl . '/hashfiles';

while (defined($next_hash_file = <$hash_files/*.pl>)) {
require $next_hash_file;
}

@months = ("January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December");

#%cardinals = ("one" => "1", "two" => "2", "three" => "3", "four" => "4", "five" => "5", "six" => "6", "seven" => "7", "eight" => "8", "nine" => "9", "ten" => "10", "eleven" => "11", "twelve" => "12", "thirteen" => "13", "fourteen" => "14", "fifteen" => "15", "sixteen" => "16", "seventeen" => "17", "eighteen" => "18", "nineteen" => "19", "twenty" => "20", "thirty" => "30", "forty" => "40", "fifty" => "50", "sixty" => "60", "seventy" => "70", "eighty" => "80", "ninety" => "90");
#@cardinalskeys = keys %cardinals;
#$cardinals_keys = join("|", @cardinalskeys);
#
#%ordinals = ("first" => "1", "second" => "2", "third" => "3", "fourth" => "4", "fifth" => "5", "sixth" => "6", "seventh" => "7", "eighth" => "8", "ninth" => "9", "tenth" => "10", "eleventh" => "11", "twelfth" => "12", "thirteenth" => "13", "fourteenth" => "14", "fifteenth" => "15", "sixteenth" => "16", "seventeenth" => "17", "eighteenth" => "18", "nineteenth" => "19", "twentieth" => "20", "thirtieth" => "30", "fortieth" => "40", "fiftieth" => "50", "sixtieth" => "60", "seventieth" => "70", "eightieth" => "80", "ninetieth" => "90");
#@ordinalskeys = keys %ordinals;
#$ordinals_keys = join("|", @ordinalskeys);
#
#%decades = ("tens" => "10", "twenties" => "20", "thirties" => "30", "forties" => "40", "fifties" => "50", "sixties" => "60", "seventies" => "70", "eighties" => "80", "nineties" => "90", "hundreds" => "00");
#@decadeskeys = keys %decades;
#$decades_keys = join("|", @decadeskeys);
#
#%places = ("hundred" => "100", "thousand" => "1000");
#@placeskeys = keys %places;
#$places_keys = join("|", @placeskeys);

%alphanumarray = ("0" => "00", "1" => "01", "2" => "02", "3" => "03", "4" => "04", "5" => "05", "6" => "06", "7" => "07", "8" => "08", "9" => "09", "A" => "10", "B" => "11", "C" => "12", "D" => "13", "E" => "14", "F" => "15", "G" => "16", "H" => "17", "I" => "18", "J" => "19", "K" => "20", "L" => "21", "M" => "22", "N" => "23", "O" => "24", "P" => "25", "Q" => "26", "R" => "27", "S" => "28", "T" => "29", "U" => "30", "V" => "31", "W" => "32", "X" => "33", "Y" => "34", "Z" => "35");

%superscripts = ("0" => "\x{2070}", "1" => "\xB9", "2" => "\xB2", "3" => "\xB3", "4" => "\x{2074}", "5" => "\x{2075}", "6" => "\x{2076}", "7" => "\x{2077}", "8" => "\x{2078}", "9" => "\x{2079}", "+" => "\x{207A}", "-" => "\x{207B}", "(" => "\x{207D}", ")" => "\x{207E}");
%subscripts = ("0" => "\x{2080}", "1" => "\x{2081}", "2" => "\x{2082}", "3" => "\x{2083}", "4" => "\x{2084}", "5" => "\x{2085}", "6" => "\x{2086}", "7" => "\x{2087}", "8" => "\x{2088}", "9" => "\x{2089}", "+" => "\x{208A}", "-" => "\x{208B}", "(" => "\x{208D}", ")" => "\x{208E}");
%greekscripts = ("s" => "\x{03c0}", "{" => "\x{03c7}", "\x{DC}" => "\x{03B1}", "\x{DD}" => "\x{03B2}", "\x{DE}" => "\x{03B3}", "\x{DF}" => "\x{00DF}");
%otherletters = ("\x{BF}" => "\x{00BF}", "\x{C7}" => "\x{00DF}", "\x{DC}" => "[alpha]", "\x{DD}" => "[beta]", "\x{DE}" => "[gamma]", "\x{DF}" => "\x{00DF}");

if($consortium ne '') {
@MeSH_major_players = keys %MeSH_major_players_hash;
(@other_MeSH_major_players_set = @MeSH_major_players) =~ s/$institution//;
$other_MeSH_major_players_set_string = join("', '", @other_MeSH_major_players_set);

@major_players = keys %major_players_hash;
(@other_major_players_set = @major_players) =~ s/$institution//;
$other_major_players_set_string = join("', '", @other_major_players_set);

@consortium_members = keys %consortium_members_hash;
(@other_consortium_members_set = @consortium_members) =~ s/$institution//;
$other_consortium_members_set_string = join("', '", @other_consortium_members_set);

}

$record_separator = '\\n';

use DBI;

# connection for the select
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host", $db_user, $db_pass, { mysql_enable_utf8 =>1 }) or die "couldn't connect to database: " . DBI->errstr;

for($k = 10; $k < 36; $k++) {
        $range = $k . "%";
        $l = 0;
#print "k: " . $k . " => range: " . $range . "\n";
        $querystring = "SELECT * FROM $db_local_tables_hash{'lc_classes_table'}
            WHERE range1 LIKE ? ORDER BY range1, range2 DESC";

        $query = $dbh->prepare($querystring)
            or die("couldn't prepare statement: ". $dbh->errstr);

        $query->execute($range)
            or die("couldn't execute statement: ". $dbh->errstr);

#print "number of alllcclasses: " . $query->rows . "\n";

        while (@row = $query->fetchrow_array) {
            ($alpha_start = @row[0]) =~ s/ //g;
            ($class_start = @row[1]) =~ s/ //g;
            ($alpha_end = @row[2]) =~ s/ //g;
            ($class_end = @row[3]) =~ s/ //g;
            if(($class_start == 1) && ($class_end =~ /^9999/)) {
                $this_lcc_label = "$alpha_start - @row[4]";
            } else {
                $this_lcc_label = "$alpha_start$class_start - $alpha_end$class_end  @row[4]";
            }

            $range1 = substr(@row[5], 0, 10) . "." . substr(@row[5], 10);
            $range2 = substr(@row[6], 0, 10) . "." . substr(@row[6], 10);
            $range2 =~ s/9999999$/0000000/;
#print "kl: " . $k . "-" . $l . ": " . $this_lcc_label . "--" . $range1 . "-" . $range2 . "\n";

            push @{ $alllcclasses_array[$k][$l]}, "$this_lcc_label", "$range1", "$range2";

            $l++;
        }
}

$query->finish();

$dbh->disconnect;


}

sub uniqueness {
    $label = $_[1];
    ($content = $_[2]) =~ s/\.,[\s]*$|,[\s]*$//g;

    $term = join "", $label, $content;
    
    if($query_insert{$_[0]} !~ /(^|\Q$record_separator\E)\Q$term\E($|\Q$record_separator\E)/) {
        if($query_insert{$_[0]} ne '') {
            $query_insert{$_[0]} .= $record_separator;
        }

        $query_insert{$_[0]} .= $term;
    }
}


sub uniqueness_diacritics {
    $label = $_[1];
    ($content = $_[2]) =~ s/\.[\s]*$|\.,[\s]*$|,[\s]*$//g;

    $term = '';

    if($label eq '') {
    $term = join '', $label, $content;
    } else {
    $label =~ s/ $//;
    $term = join '\\t', $label, $content;
    }

#print $term . "\n";

    while($term =~ /([\xDC-\xDF])/) {
        $term =~ s/$1/$greekscripts{$1}/g;
    }

    if($term =~ /[\xA0-\xFF]/) {
       $term =  marc8_to_utf8($term, 'ignore-errors');
        while($term =~ /([\0]|[\x{FE21}\x{FE23}])/) {
            $term =~ s/$1//g;
        }
    }

    if($term =~ /[\x1b]p([0-9\+\-\(\)]*)[\x1b]s/) {
#print $this_local_control_number . " has superscript characters:" . $term . "\n";
        foreach $power ($term =~ /[\x1b]p([0-9\+\-\(\)]*)[\x1b]s/g) {
            for($j = 0; $j < length($power); $j++) {
                $this_power = substr($power, $j, 1);
                $all_powers .= $superscripts{$this_power};
            }
            $power =~ s/\+/[+]/g;
            $power =~ s/\-/[-]/g;
            $power =~ s/\(/[(]/g;
            $power =~ s/\)/[)]/g;

        $term =~ s/[\x1b]p$power[\x1b]s/$all_powers/g;
#print $term . "\n";
    undef($all_powers);
    }
    }

    if($term =~ /[\x1b]b([0-9\+\-\(\)]*)[\x1b]s/) {
#print $this_local_control_number . " has subscript characters:" . $term . "\n";
        foreach $power ($term =~ /[\x1b]b([0-9\+\-\(\)]*)[\x1b]s/g) {
            for($j = 0; $j < length($power); $j++) {
                $this_power = substr($power, $j, 1);
                $all_powers .= $subscripts{$this_power};
            }
            $power =~ s/\+/[+]/g;
            $power =~ s/\-/[-]/g;
            $power =~ s/\(/[(]/g;
            $power =~ s/\)/[)]/g;

        $term =~ s/[\x1b]b$power[\x1b]s/$all_powers/g;
#print $term . "\n";
    undef($all_powers);
    }
    }

    if($term =~ /[\x1b]g([s{])[\x1b]s/) {
        while($term =~ /[\x1b]g([s{])[\x1b]s/) {
            $power = $1;
            $term =~ s/[\x1b]g$power[\x1b]s/$greekscripts{$power}/g;
        }
    }

    if($term =~ /[\x1b]\(Q[AS]*[ ]*[\x1b]\(B/) {
        foreach $single_quote ($term =~ /([\x1b]\(Q[AS]*[ ]*[\x1b]\(B)/g) {
            print $this_local_control_number . " has escaped single quote characters ($single_quote)in this string: " . $term . "\n";
            $term =~ s/\Q$single_quote\E/'/g;
            print  $term . "\n";
        }
    }

    if($term =~ /\x1a/) {
#        print $this_local_control_number . " has an escape character (either a tab or line ending) in a field marked with \x1a \n";
        $term =~ s/\x1a//g;
    }

    if($term =~ /[\x1b]|[\x1d-\x1f]]|[\x4-\x5]/) {
        foreach $bad_character ($term =~ /([\x1b]|[\x1d-\x1f]]|[\x4-\x5])/g) {
            print "$this_local_control_number has a bad character ($bad_character) in this string: \n$term\n\n";
                $term =~ s/$bad_character//g;
            print "$term\n";
        }
    }

    if($term =~ /\\[a-su-zA-SU-Z0-9]/) {
        foreach $bad_escape ($term =~ /\\([a-su-zA-SU-Z0-9])/g) {
            print "$this_local_control_number has a bad escape character ($bad_escape) in this string: \n$term\n\n";
                $term =~ s/\\$bad_escape/$bad_escape/g;
            print "$term\n";
        }
    }

        $term = NFC($term);

#print $term . "\n";

if($_[0] =~ /raw_tracing/) {

    push(@raw_tracings_array, $term);

} else {
    if($query_insert{$_[0]} !~ /(^|\Q$record_separator\E)\Q$term\E($|\Q$record_separator\E)/) {
        if($query_insert{$_[0]} ne '') {
            $query_insert{$_[0]} .= $record_separator;
        }

#print $term . "\n";

        $query_insert{$_[0]} .= $term;
    }
}
}

sub dates_expansion {
    $this_tag = $_[0];
    $thesaurus = $_[1];
    $date_data = $_[2];

    uniqueness_diacritics($this_tag, $thesaurus, $date_data);

    if($date_data !~ /\bTo [0-9]{1,4}|century|centuries/) {

        if($date_data =~ /[0-9]/) {

        my @this_year = ();
        my @final_year = ();
        my @decades = ();
        my @centuries = ();
        my @eras = ();
        my $years_phrase = '';
        my $decades_phrase = '';
        my $century_phrase = '';
        my @quotient = ();

    if($date_data =~ /([0-9]{1,4})[ ]*B.C[\.]*-([0-9]{1,4})[ ]*A.D/) {
$eras[0] = ' B.C.';
$eras[1] = ' A.D.';
$this_year[0] = $1;
$this_year[1] = $2;
($decades[0] = $this_year[0]) =~ s/[1-8]$/9/;
($decades[1] = $this_year[1]) =~ s/[1-8]$/9/;
    } elsif($date_data =~ /([0-9]{1,4})([ ]*B.C[\.]*)*-([0-9]{1,4})[ ]*B.C/) {
$eras[0] = ' B.C.';
$eras[1] = ' B.C.';
$this_year[0] = $1;
$this_year[1] = $3;
($decades[0] = $this_year[0]) =~ s/[1-8]$/9/;
($decades[1] = $this_year[1]) =~ s/[1-9]$/0/;
    } elsif($date_data =~ /([0-9]{1,4})[ ]*B.C/) {
$eras[0] = '';
$eras[1] = ' B.C.';
$this_year[0] = $1;
$this_year[1] = $1;
($decades[0] = $this_year[0]) =~ s/[0-8]$/9/;
($decades[1] = $this_year[1]) =~ s/[1-9]$/0/;
    } elsif($date_data =~ /([0-9]{1,4})([ ]*A.D[\.]*)*-([0-9]{1,4})([ ]*A.D[\.]*)*/) {
$eras[0] = '';
$eras[1] = '';
$this_year[0] = $1;
$this_year[1] = $3;
($decades[0] = $this_year[0]) =~ s/[1-9]$/0/;
($decades[1] = $this_year[1]) =~ s/[1-8]$/9/;
    } elsif($date_data =~ /([0-9]{1,4})([ ]*A.D[\.]*)*/) {
$eras[0] = '';
$eras[1] = '';
$this_year[0] = $1;
$this_year[1] = $1;
($decades[0] = $this_year[0]) =~ s/[1-9]$/0/;
($decades[1] = $this_year[1]) =~ s/[0-8]$/9/;
}

#print "$this_local_control_number: $date_data \n";

#print "this_year0: $this_year[0] -- this_year1: $this_year[1]\n";

        if($#this_year == 0) {
            $decades_phrase = $decades[0] . $eras[0] . '-' . $decades[1] . $eras[1];
            $years_phrase = $this_year[0] . $eras[0];
        } else {
            if(($this_year[0] eq $this_year[1]) && ($eras[0] eq $eras[1])){
                $years_phrase = $this_year[0] . $eras[1];
                $decades_phrase = $decades[0] . '-' . $decades[1] . $eras[1];
            } elsif(($this_year[0] ne $this_year[1]) && ($eras[0] eq $eras[1])){
                $years_phrase = $this_year[0] . '-' . $this_year[1] . $eras[1];
                $decades_phrase = $decades[0] . '-' . $decades[1] . $eras[1];
            } else {
                $years_phrase = $this_year[0] . $eras[0] . '-' . $this_year[1] . $eras[1];
                $decades_phrase = $decades[0] . $eras[0] . '-' . $decades[1] . $eras[1];
            }
        }

#print "$decades \n";

        for($i = 0; $i <= $#this_year; $i++) {

            $quotient[$i] = $this_year[$i] / 100;

            if($i == 0) {
                if (($#this_year > 0) && ($quotient[$i] eq int($quotient[$i]))) {
                    $quotient[$i] += 1;
                }
            }

            if($i == 1) {
                if($quotient[1] < $quotient[0]) {
                    $quotient[1] = $quotient[0];
                }
            }

            $quotient[$i] = ceil($quotient[$i]);

            $part = substr $quotient[$i], -2;

            if($part =~ /[02-9]1|^1$/) {
                $suffix = "st";
            } elsif ($part =~ /[02-9]2|^2$/) {
                $suffix = "nd";
            } elsif ($part =~ /[02-9]3|^3$/) {
                $suffix = "rd";
            } else {
                $suffix = "th";
            }


            push(@centuries, $quotient[$i] . $suffix);

        }

        if($#centuries > 0) {
            if(($centuries[0] eq $centuries[1]) && ($eras[0] eq $eras[1])){
                $century_phrase = $centuries[0] . ' century' . $eras[1];
            } elsif(($centuries[0] ne $centuries[1]) && ($eras[0] eq $eras[1])){
                $century_phrase = $centuries[0] . '-' . $centuries[1] . ' centuries' . $eras[1];
            } else {
                $century_phrase = $centuries[0] . ' century' . $eras[0] . '-' . $centuries[1] . ' century' . $eras[1];
            }
        } else {
            $century_phrase = $centuries[0] . ' century' .  $eras[1];
        }

    uniqueness_diacritics($this_tag, $thesaurus, $years_phrase);
    uniqueness_diacritics($this_tag, $thesaurus, $decades_phrase);
    uniqueness_diacritics($this_tag, $thesaurus, $century_phrase);

#print "$date_data -- $decades_phrase -- $century_phrase\n\n";

        }
    }

#                    if($date_data =~ /([0-9]{1,2})([a-zA-Z]{2}) century/) {
#                        $this_century1 = ($1 - 1) * 100;
#                        $this_century2 = $this_century1 + 99;
#                        uniqueness($this_tag, $thesaurus . '\\t', "$this_century1-$this_century2");
#                    }
#
#                if($date_data !~ /\bTo [0-9]{1,4}/) {
#
#                    if($date_data =~ /\bA.D\b/) {
#
#                        foreach $i ($date_data =~ /\b[0-9]* B.C.-[0-9]* A.D\b/g) {
#                            uniqueness($this_tag, $thesaurus . '\\t', $i);
#                            if($i =~ /([0-9]{1,4}) B.C.-([0-9]{1,4}) A.D/) {
#                                $date1 = $1;
#                                $date2 = $2;
#                            }
#
#                            uniqueness($this_tag, $thesaurus . '\\t', "$date1 B.C-$date2 A.D");
#                            ($decade1 = $date1) =~ s/[0-8]$/9/;
#                            ($decade2 = $date2) =~ s/[0-8]$/9/;
#                            uniqueness($this_tag, $thesaurus . '\\t', "$decade1 B.C-$decade2 A.D");
#
#                            if($decade1 > 0) {
#                                $decades_begin = $decade1;
#                                while($decades_begin > 0) {
#                                    ($decades_end = $decades_begin) =~ s/9$/0/;
#                                    $decades_end =~ s/^0$/1/;
#                                    uniqueness($this_tag, $thesaurus . '\\t', "$decades_begin B.C-$decades_end B.C");
#                                    $decades_begin -= 10;
#                                }
#                            }
#
#                            if($decade2 > 1) {
#                                $decades_begin = 1;
#                                while($decades_begin < $decade2) {
#                                    ($decades_end = $decades_begin) =~ s/0$|1$/9/;
#                                    uniqueness($this_tag, $thesaurus . '\\t', "$decades_begin-$decades_end");
#                                    $decades_begin =~ s/^1$/0/;
#                                    $decades_begin += 10;
#                                }
#                            }
#
#                            ($century1 = $decade1) =~ s/[0-8]9$/99/;
#                            ($century2 = $decade2) =~ s/[1-9]0$/00/;
#                            if($century1 > 0) {
#                                $centurys_begin = $century1;
#                                while($centurys_begin > 0) {
#                                    ($centurys_end = $centurys_begin) =~ s/99$/00/;
#                                    $centurys_end =~ s/^00$/1/;
#                                    uniqueness($this_tag, $thesaurus . '\\t', "$centurys_begin B.C-$centurys_end B.C");
#                                    $centurys_begin -= 100;
#                                }
#                            }
#                            if($century2 > 1) {
#                                $centurys_begin = 1;
#                                while($centurys_begin < $century2) {
#                                    ($centurys_end = $centurys_begin) =~ s/00$|1$/99/;
#                                    uniqueness($this_tag, $thesaurus . '\\t', "$centurys_begin-$centurys_end");
#                                    $centurys_begin =~ s/^1$/0/;
#                                    $centurys_begin += 100;
#                                }
#                            }
#
#
#                        }
#
#                    } elsif($date_data =~ /\bB.C\b/) {
#
#                        foreach $i ($date_data =~ /\b[0-9]{1,4} B.C.-[0-9]{1,4} B.C\b|\b[0-9]{1,4}-[0-9]{1,4} B.C\b|\b[0-9]{1,4} B.C\b/g) {
#                            uniqueness($this_tag, $thesaurus . '\\t', $i);
#                            if($i =~ /([0-9]{1,4}) B.C.-([0-9]{1,4}) B.C/) {
#                                $date1 = $1;
#                                $date2 = $2;
#                            } elsif ($i =~ /([0-9]{1,4})-([0-9]{1,4}) B.C/) {
#                                $date1 = $1;
#                                $date2 = $2;
#                            } elsif ($i =~ /([0-9]{1,4}) B.C/) {
#                                $date1 = $1;
#                                $date2 = $1;
#                            }
#
#                            uniqueness($this_tag, $thesaurus . '\\t', "$date1 B.C-$date2 B.C");
#                            ($decade1 = $date1) =~ s/[0-8]$/9/;
#                            ($decade2 = $date2) =~ s/[1-9]$/0/;
#                            uniqueness($this_tag, $thesaurus . '\\t', "$decade1 B.C-$decade2 B.C");
#                            if($decade1 > $decade2) {
#                                $decades_begin = $decade1;
#                                while($decades_begin > $decade2) {
#                                    ($decades_end = $decades_begin) =~ s/9$/0/;
#                                    $decades_end =~ s/^0$/1/;
#                                    uniqueness($this_tag, $thesaurus . '\\t', "$decades_begin B.C-$decades_end B.C");
#                                    $decades_begin -= 10;
#                                }
#                            }
#
#                            ($century1 = $decade1) =~ s/[0-8]9$/99/;
#                            ($century2 = $decade2) =~ s/[1-9]0$/00/;
#                            uniqueness($this_tag, $thesaurus . '\\t', "$century1 B.C-$century2 B.C");
#                            if($century1 > $century2) {
#                                $centurys_begin = $century1;
#                                while($centurys_begin > $century2) {
#                                    ($centurys_end = $centurys_begin) =~ s/99$/00/;
#                                    $centurys_end =~ s/^00$/1/;
#                                    uniqueness($this_tag, $thesaurus . '\\t', "$centurys_begin B.C-$centurys_end B.C");
#                                    $centurys_begin -= 100;
#                                }
#                            }
#
#                        }
#
#                    } else {
#                        foreach $i ($date_data =~ /\b[0-9]{1,4}-[0-9]{1,4}\b|\b[0-9]{1,4}-|\b[0-9]{1,4}\b/g) {
#                            if($i =~ /([0-9]{1,4})-([0-9]{1,4})/) {
#                                $date1 = $1;
#                                $date2 = $2;
#                            } elsif ($i =~ /([0-9]{1,4})-/) {
#                                $date1 = $1;
#                                $date2 = $query_insert{'original_pubdate_end'};
#                            } else {
#                                $date1 = $i;
#                                $date2 = $i;
#                            }
#
#                            uniqueness($this_tag, $thesaurus . '\\t', "$date1-$date2");
#                            ($decade1 = $date1) =~ s/[1-9]$/0/;
#                            ($decade2 = $date2) =~ s/[0-8]$/9/;
#                            uniqueness($this_tag, $thesaurus . '\\t', "$decade1-$decade2");
#                            if($decade1 < $decade2) {
#                                $decades_begin = $decade1;
#                                while($decades_begin < $decade2) {
#                                    ($decades_end = $decades_begin) =~ s/0$|1$/9/;
#                                    uniqueness($this_tag, $thesaurus . '\\t', "$decades_begin-$decades_end");
#                                    $decades_begin += 10;
#                                }
#                            }
#
#                            ($century1 = $decade1) =~ s/[1-9]0$/00/;
#                            ($century2 = $decade2) =~ s/[0-8]9$/99/;
#                            uniqueness($this_tag, $thesaurus . '\\t', "$century1-$century2");
#                            if($century1 < $century2) {
#                                $centurys_begin = $century1;
#                                while($centurys_begin < $century2) {
#                                    ($centurys_end = $centurys_begin) =~ s/00$|1$/99/;
#                                    uniqueness($this_tag, $thesaurus . '\\t', "$centurys_begin-$centurys_end");
#                                    $centurys_begin += 100;
#                                }
#                            }
#
#                        }
#
#                    }
#
#            }

}

